Index: /trunk/src/VBox/VMM/PDMAsyncCompletionFile.cpp =================================================================== --- /trunk/src/VBox/VMM/PDMAsyncCompletionFile.cpp (revision 26670) +++ /trunk/src/VBox/VMM/PDMAsyncCompletionFile.cpp (revision 26671) @@ -464,4 +464,31 @@ /** + * I/O refresh timer callback. + */ +static void pdmacFileBwRefresh(PVM pVM, PTMTIMER pTimer, void *pvUser) +{ + PPDMACFILEBWMGR pBwMgr = (PPDMACFILEBWMGR)pvUser; + + LogFlowFunc(("pVM=%p pTimer=%p pvUser=%p\n", pVM, pTimer, pvUser)); + + /* Reset the counter growing the maximum if allowed and needed */ + bool fIncreaseNeeded = ASMAtomicReadBool(&pBwMgr->fVMTransferLimitReached); + + if ( fIncreaseNeeded + && pBwMgr->cbVMTransferPerSecStart < pBwMgr->cbVMTransferPerSecMax) + { + pBwMgr->cbVMTransferPerSecStart = RT_MIN(pBwMgr->cbVMTransferPerSecMax, pBwMgr->cbVMTransferPerSecStart + pBwMgr->cbVMTransferPerSecStep); + LogFlow(("AIOMgr: Increasing maximum bandwidth to %u bytes/sec\n", pBwMgr->cbVMTransferPerSecStart)); + } + + /* Update */ + ASMAtomicWriteU32(&pBwMgr->cbVMTransferAllowed, pBwMgr->cbVMTransferPerSecStart); + ASMAtomicWriteBool(&pBwMgr->fVMTransferLimitReached, false); + + /* Arm the timer */ + TMTimerSetMillies(pTimer, 1000); +} + +/** * Destroys a async I/O manager. * @@ -490,5 +517,4 @@ pEpClassFile->cAioMgrs--; - rc = RTCritSectLeave(&pEpClassFile->CritSect); AssertRC(rc); @@ -501,4 +527,83 @@ MMR3HeapFree(pAioMgr); +} + +static int pdmacFileBwMgrInitialize(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile, + PCFGMNODE pCfgNode, PPPDMACFILEBWMGR ppBwMgr) +{ + int rc = VINF_SUCCESS; + PPDMACFILEBWMGR pBwMgr = NULL; + + rc = MMR3HeapAllocZEx(pEpClassFile->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION, + sizeof(PDMACFILEBWMGR), + (void **)&pBwMgr); + if (RT_SUCCESS(rc)) + { + /* Init I/O flow control. */ + rc = CFGMR3QueryU32Def(pCfgNode, "VMTransferPerSecMax", &pBwMgr->cbVMTransferPerSecMax, UINT32_MAX); + AssertLogRelRCReturn(rc, rc); + rc = CFGMR3QueryU32Def(pCfgNode, "VMTransferPerSecStart", &pBwMgr->cbVMTransferPerSecStart, _1M); + AssertLogRelRCReturn(rc, rc); + rc = CFGMR3QueryU32Def(pCfgNode, "VMTransferPerSecStep", &pBwMgr->cbVMTransferPerSecStep, _1M); + AssertLogRelRCReturn(rc, rc); + + pBwMgr->cbVMTransferAllowed = pBwMgr->cbVMTransferPerSecStart; + + /* Init the refresh timer */ + rc = TMR3TimerCreateInternal(pEpClassFile->Core.pVM, + TMCLOCK_REAL, + pdmacFileBwRefresh, + pBwMgr, + "AsyncCompletionFile-BW-Refresh", + &pBwMgr->pBwRefreshTimer); + if (RT_SUCCESS(rc)) + *ppBwMgr = pBwMgr; + else + MMR3HeapFree(pBwMgr); + } + + return rc; +} + +static void pdmacFileBwMgrDestroy(PPDMACFILEBWMGR pBwMgr) +{ + TMR3TimerDestroy(pBwMgr->pBwRefreshTimer); + MMR3HeapFree(pBwMgr); +} + +static void pdmacFileBwRef(PPDMACFILEBWMGR pBwMgr) +{ + pBwMgr->cRefs++; + if (pBwMgr->cRefs == 1) + TMTimerSetMillies(pBwMgr->pBwRefreshTimer, 1000); /* 1sec update interval */ +} + +static void pdmacFileBwUnref(PPDMACFILEBWMGR pBwMgr) +{ + Assert(pBwMgr->cRefs > 0); + pBwMgr->cRefs--; + if (!pBwMgr->cRefs) + TMTimerStop(pBwMgr->pBwRefreshTimer); +} + +bool pdmacFileBwMgrIsTransferAllowed(PPDMACFILEBWMGR pBwMgr, uint32_t cbTransfer) +{ + bool fAllowed = false; + + LogFlowFunc(("pBwMgr=%p cbTransfer=%u\n", pBwMgr, cbTransfer)); + + uint32_t cbOld = ASMAtomicSubU32(&pBwMgr->cbVMTransferAllowed, cbTransfer); + if (RT_LIKELY(cbOld >= cbTransfer)) + fAllowed = true; + else + { + /* We are out of ressources */ + ASMAtomicAddU32(&pBwMgr->cbVMTransferAllowed, cbTransfer); + ASMAtomicXchgBool(&pBwMgr->fVMTransferLimitReached, true); + } + + LogFlowFunc(("fAllowed=%RTbool\n", fAllowed)); + + return fAllowed; } @@ -572,4 +677,8 @@ else LogRel(("AIOMgr: Cache was globally disabled\n")); + + rc = pdmacFileBwMgrInitialize(pEpClassFile, pCfgNode, &pEpClassFile->pBwMgr); + if (RT_FAILURE(rc)) + RTCritSectDelete(&pEpClassFile->CritSect); } @@ -593,4 +702,5 @@ RTCritSectDelete(&pEpClassFile->CritSect); + pdmacFileBwMgrDestroy(pEpClassFile->pBwMgr); } @@ -705,5 +815,7 @@ pEpFile->pTasksFreeTail = pEpFile->pTasksFreeHead; - pEpFile->cTasksCached = 0; + pEpFile->cTasksCached = 0; + pEpFile->pBwMgr = pEpClassFile->pBwMgr; + pdmacFileBwRef(pEpFile->pBwMgr); if (fUseFailsafeManager) @@ -753,4 +865,5 @@ RTMemFree(pEpFile->AioMgr.pTreeRangesLocked); MMR3HeapFree(pEpFile->pTasksFreeHead); + pdmacFileBwUnref(pEpFile->pBwMgr); } } @@ -815,4 +928,7 @@ if (pEpFile->fCaching) pdmacFileEpCacheDestroy(pEpFile); + + /* Remove from the bandwidth manager */ + pdmacFileBwUnref(pEpFile->pBwMgr); /* Destroy the locked ranges tree now. */ Index: /trunk/src/VBox/VMM/PDMAsyncCompletionFileInternal.h =================================================================== --- /trunk/src/VBox/VMM/PDMAsyncCompletionFileInternal.h (revision 26670) +++ /trunk/src/VBox/VMM/PDMAsyncCompletionFileInternal.h (revision 26671) @@ -25,4 +25,5 @@ #include #include +#include #include #include @@ -148,4 +149,6 @@ /** Size of the array. */ unsigned cReqEntries; + /** Flag whether at least one endpoint reached its bandwidth limit. */ + bool fBwLimitReached; /** Critical section protecting the blocking event handling. */ RTCRITSECT CritSectBlockingEvent; @@ -188,4 +191,30 @@ /** + * Bandwidth control manager instance data + */ +typedef struct PDMACFILEBWMGR +{ + /** Maximum number of bytes the VM is allowed to transfer (Max is 4GB/s) */ + uint32_t cbVMTransferPerSecMax; + /** Number of bytes we start with */ + uint32_t cbVMTransferPerSecStart; + /** Step after each update */ + uint32_t cbVMTransferPerSecStep; + /** Number of bytes we are allowed to transfer till the next update. + * Resetted by the refresh timer. */ + volatile uint32_t cbVMTransferAllowed; + /** Flag whether a request could not processed due to the limit. */ + volatile bool fVMTransferLimitReached; + /** Reference counter - How many endpoints are associated with this manager. */ + uint32_t cRefs; + /** The refresh timer */ + PTMTIMERR3 pBwRefreshTimer; +} PDMACFILEBWMGR; +/** Pointer to a bandwidth control manager */ +typedef PDMACFILEBWMGR *PPDMACFILEBWMGR; +/** Pointer to a bandwidth control manager pointer */ +typedef PPDMACFILEBWMGR *PPPDMACFILEBWMGR; + +/** * A file access range lock. */ @@ -371,4 +400,6 @@ /** Flag whether the out of resources warning was printed already. */ bool fOutOfResourcesWarningPrinted; + /** The global bandwidth control manager */ + PPDMACFILEBWMGR pBwMgr; } PDMASYNCCOMPLETIONEPCLASSFILE; /** Pointer to the endpoint class data. */ @@ -446,4 +477,6 @@ /** Cache of endpoint data. */ PDMACFILEENDPOINTCACHE DataCache; + /** Pointer to the associated bandwidth control manager */ + PPDMACFILEBWMGR pBwMgr; /** Flag whether a flush request is currently active */ @@ -594,4 +627,6 @@ void pdmacFileEpTaskCompleted(PPDMACTASKFILE pTask, void *pvUser); +bool pdmacFileBwMgrIsTransferAllowed(PPDMACFILEBWMGR pBwMgr, uint32_t cbTransfer); + int pdmacFileCacheInit(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile, PCFGMNODE pCfgNode); void pdmacFileCacheDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile); Index: /trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp =================================================================== --- /trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp (revision 26670) +++ /trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp (revision 26671) @@ -712,4 +712,10 @@ PPDMACTASKFILE pCurr = pTaskHead; + if (!pdmacFileBwMgrIsTransferAllowed(pEndpoint->pBwMgr, pCurr->DataSeg.cbSeg)) + { + pAioMgr->fBwLimitReached = true; + break; + } + pTaskHead = pTaskHead->pNext; @@ -777,5 +783,7 @@ pdmacFileAioMgrEpAddTaskList(pEndpoint, pTaskHead); - if (RT_UNLIKELY(!cMaxRequests && !pEndpoint->pFlushReq)) + if (RT_UNLIKELY( !cMaxRequests + && !pEndpoint->pFlushReq + && !pAioMgr->fBwLimitReached)) { /* @@ -954,4 +962,6 @@ PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead; + pAioMgr->fBwLimitReached = false; + while (pEndpoint) { @@ -992,4 +1002,156 @@ return rc; +} + +static void pdmacFileAioMgrNormalReqComplete(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq) +{ + int rc = VINF_SUCCESS; + PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint; + size_t cbTransfered = 0; + int rcReq = RTFileAioReqGetRC(hReq, &cbTransfered); + PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(hReq); + + pEndpoint = pTask->pEndpoint; + + /* + * It is possible that the request failed on Linux with kernels < 2.6.23 + * if the passed buffer was allocated with remap_pfn_range or if the file + * is on an NFS endpoint which does not support async and direct I/O at the same time. + * The endpoint will be migrated to a failsafe manager in case a request fails. + */ + if (RT_FAILURE(rcReq)) + { + /* Free bounce buffers and the IPRT request. */ + pAioMgr->pahReqsFree[pAioMgr->iFreeEntryNext] = hReq; + pAioMgr->iFreeEntryNext = (pAioMgr->iFreeEntryNext + 1) % pAioMgr->cReqEntries; + + pAioMgr->cRequestsActive--; + pEndpoint->AioMgr.cRequestsActive--; + pEndpoint->AioMgr.cReqsProcessed++; + + if (pTask->fBounceBuffer) + RTMemFree(pTask->pvBounceBuffer); + + /* Queue the request on the pending list. */ + pTask->pNext = pEndpoint->AioMgr.pReqsPendingHead; + pEndpoint->AioMgr.pReqsPendingHead = pTask; + + /* Create a new failsafe manager if neccessary. */ + if (!pEndpoint->AioMgr.fMoving) + { + PPDMACEPFILEMGR pAioMgrFailsafe; + + LogRel(("%s: Request %#p failed with rc=%Rrc, migrating endpoint %s to failsafe manager.\n", + RTThreadGetName(pAioMgr->Thread), pTask, rcReq, pEndpoint->Core.pszUri)); + + pEndpoint->AioMgr.fMoving = true; + + rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass, + &pAioMgrFailsafe, true); + AssertRC(rc); + + pEndpoint->AioMgr.pAioMgrDst = pAioMgrFailsafe; + + /* Update the flags to open the file with. Disable async I/O and enable the host cache. */ + pEndpoint->fFlags &= ~(RTFILE_O_ASYNC_IO | RTFILE_O_NO_CACHE); + } + + /* If this was the last request for the endpoint migrate it to the new manager. */ + if (!pEndpoint->AioMgr.cRequestsActive) + { + bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint); + Assert(!fReqsPending); + + rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint); + AssertRC(rc); + } + } + else + { + AssertMsg(( (cbTransfered == pTask->DataSeg.cbSeg) + || (pTask->fBounceBuffer && (cbTransfered >= pTask->DataSeg.cbSeg))), + ("Task didn't completed successfully (rc=%Rrc) or was incomplete (cbTransfered=%u)\n", rcReq, cbTransfered)); + + if (pTask->fPrefetch) + { + Assert(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE); + Assert(pTask->fBounceBuffer); + + memcpy(((uint8_t *)pTask->pvBounceBuffer) + pTask->uBounceBufOffset, + pTask->DataSeg.pvSeg, + pTask->DataSeg.cbSeg); + + /* Write it now. */ + pTask->fPrefetch = false; + size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg, 512); + RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1); + + /* Grow the file if needed. */ + if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile)) + { + ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg); + RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg); + } + + rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File, + offStart, pTask->pvBounceBuffer, cbToTransfer, pTask); + AssertRC(rc); + rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, &hReq, 1); + AssertRC(rc); + } + else + { + if (pTask->fBounceBuffer) + { + if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ) + memcpy(pTask->DataSeg.pvSeg, + ((uint8_t *)pTask->pvBounceBuffer) + pTask->uBounceBufOffset, + pTask->DataSeg.cbSeg); + + RTMemPageFree(pTask->pvBounceBuffer); + } + + /* Put the entry on the free array */ + pAioMgr->pahReqsFree[pAioMgr->iFreeEntryNext] = hReq; + pAioMgr->iFreeEntryNext = (pAioMgr->iFreeEntryNext + 1) % pAioMgr->cReqEntries; + + pAioMgr->cRequestsActive--; + pEndpoint->AioMgr.cRequestsActive--; + pEndpoint->AioMgr.cReqsProcessed++; + + /* Free the lock and process pending tasks if neccessary */ + pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock); + + /* Call completion callback */ + pTask->pfnCompleted(pTask, pTask->pvUser); + pdmacFileTaskFree(pEndpoint, pTask); + + /* + * If there is no request left on the endpoint but a flush request is set + * it completed now and we notify the owner. + * Furthermore we look for new requests and continue. + */ + if (!pEndpoint->AioMgr.cRequestsActive && pEndpoint->pFlushReq) + { + /* Call completion callback */ + pTask = pEndpoint->pFlushReq; + pEndpoint->pFlushReq = NULL; + + AssertMsg(pTask->pEndpoint == pEndpoint, ("Endpoint of the flush request does not match assigned one\n")); + + pTask->pfnCompleted(pTask, pTask->pvUser); + pdmacFileTaskFree(pEndpoint, pTask); + } + else if (RT_UNLIKELY(!pEndpoint->AioMgr.cRequestsActive && pEndpoint->AioMgr.fMoving)) + { + /* If the endpoint is about to be migrated do it now. */ + bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint); + Assert(!fReqsPending); + + rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint); + AssertRC(rc); + } + } + } /* request completed successfully */ } @@ -1040,202 +1202,66 @@ CHECK_RC(pAioMgr, rc); - while (pAioMgr->cRequestsActive) - { - RTFILEAIOREQ apReqs[20]; - uint32_t cReqsCompleted = 0; - size_t cReqsWait; - - if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs)) - cReqsWait = RT_ELEMENTS(apReqs); + while ( pAioMgr->cRequestsActive + || pAioMgr->fBwLimitReached) + { + if (pAioMgr->cRequestsActive) + { + RTFILEAIOREQ apReqs[20]; + uint32_t cReqsCompleted = 0; + size_t cReqsWait; + + if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs)) + cReqsWait = RT_ELEMENTS(apReqs); + else + cReqsWait = pAioMgr->cRequestsActive; + + LogFlow(("Waiting for %d of %d tasks to complete\n", pAioMgr->cRequestsActive, cReqsWait)); + + rc = RTFileAioCtxWait(pAioMgr->hAioCtx, + cReqsWait, + RT_INDEFINITE_WAIT, apReqs, + RT_ELEMENTS(apReqs), &cReqsCompleted); + if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED)) + CHECK_RC(pAioMgr, rc); + + LogFlow(("%d tasks completed\n", cReqsCompleted)); + + for (uint32_t i = 0; i < cReqsCompleted; i++) + pdmacFileAioMgrNormalReqComplete(pAioMgr, apReqs[i]); + + /* Check for an external blocking event before we go to sleep again. */ + if (pAioMgr->fBlockingEventPending) + { + rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr); + CHECK_RC(pAioMgr, rc); + } + + /* Update load statistics. */ + uint64_t uMillisCurr = RTTimeMilliTS(); + if (uMillisCurr > uMillisEnd) + { + PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointCurr = pAioMgr->pEndpointsHead; + + /* Calculate timespan. */ + uMillisCurr -= uMillisEnd; + + while (pEndpointCurr) + { + pEndpointCurr->AioMgr.cReqsPerSec = pEndpointCurr->AioMgr.cReqsProcessed / (uMillisCurr + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD); + pEndpointCurr->AioMgr.cReqsProcessed = 0; + pEndpointCurr = pEndpointCurr->AioMgr.pEndpointNext; + } + + /* Set new update interval */ + uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD; + } + } else - cReqsWait = pAioMgr->cRequestsActive; - - LogFlow(("Waiting for %d of %d tasks to complete\n", pAioMgr->cRequestsActive, cReqsWait)); - - rc = RTFileAioCtxWait(pAioMgr->hAioCtx, - cReqsWait, - RT_INDEFINITE_WAIT, apReqs, - RT_ELEMENTS(apReqs), &cReqsCompleted); - if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED)) - CHECK_RC(pAioMgr, rc); - - LogFlow(("%d tasks completed\n", cReqsCompleted)); - - for (uint32_t i = 0; i < cReqsCompleted; i++) { - PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint; - size_t cbTransfered = 0; - int rcReq = RTFileAioReqGetRC(apReqs[i], &cbTransfered); - PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(apReqs[i]); - - pEndpoint = pTask->pEndpoint; - /* - * It is possible that the request failed on Linux with kernels < 2.6.23 - * if the passed buffer was allocated with remap_pfn_range or if the file - * is on an NFS endpoint which does not support async and direct I/O at the same time. - * The endpoint will be migrated to a failsafe manager in case a request fails. + * Bandwidth limit reached for all endpoints. + * Yield and wait until we have enough resources again. */ - if (RT_FAILURE(rcReq)) - { - /* Free bounce buffers and the IPRT request. */ - pAioMgr->pahReqsFree[pAioMgr->iFreeEntryNext] = apReqs[i]; - pAioMgr->iFreeEntryNext = (pAioMgr->iFreeEntryNext + 1) % pAioMgr->cReqEntries; - - pAioMgr->cRequestsActive--; - pEndpoint->AioMgr.cRequestsActive--; - pEndpoint->AioMgr.cReqsProcessed++; - - if (pTask->fBounceBuffer) - RTMemFree(pTask->pvBounceBuffer); - - /* Queue the request on the pending list. */ - pTask->pNext = pEndpoint->AioMgr.pReqsPendingHead; - pEndpoint->AioMgr.pReqsPendingHead = pTask; - - /* Create a new failsafe manager if neccessary. */ - if (!pEndpoint->AioMgr.fMoving) - { - PPDMACEPFILEMGR pAioMgrFailsafe; - - LogRel(("%s: Request %#p failed with rc=%Rrc, migrating endpoint %s to failsafe manager.\n", - RTThreadGetName(pAioMgr->Thread), pTask, rcReq, pEndpoint->Core.pszUri)); - - pEndpoint->AioMgr.fMoving = true; - - rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass, - &pAioMgrFailsafe, true); - AssertRC(rc); - - pEndpoint->AioMgr.pAioMgrDst = pAioMgrFailsafe; - - /* Update the flags to open the file with. Disable async I/O and enable the host cache. */ - pEndpoint->fFlags &= ~(RTFILE_O_ASYNC_IO | RTFILE_O_NO_CACHE); - } - - /* If this was the last request for the endpoint migrate it to the new manager. */ - if (!pEndpoint->AioMgr.cRequestsActive) - { - bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint); - Assert(!fReqsPending); - - rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint); - AssertRC(rc); - } - } - else - { - AssertMsg(( (cbTransfered == pTask->DataSeg.cbSeg) - || (pTask->fBounceBuffer && (cbTransfered >= pTask->DataSeg.cbSeg))), - ("Task didn't completed successfully (rc=%Rrc) or was incomplete (cbTransfered=%u)\n", rcReq, cbTransfered)); - - if (pTask->fPrefetch) - { - Assert(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE); - Assert(pTask->fBounceBuffer); - - memcpy(((uint8_t *)pTask->pvBounceBuffer) + pTask->uBounceBufOffset, - pTask->DataSeg.pvSeg, - pTask->DataSeg.cbSeg); - - /* Write it now. */ - pTask->fPrefetch = false; - size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg, 512); - RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1); - - /* Grow the file if needed. */ - if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile)) - { - ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg); - RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg); - } - - rc = RTFileAioReqPrepareWrite(apReqs[i], pEndpoint->File, - offStart, pTask->pvBounceBuffer, cbToTransfer, pTask); - AssertRC(rc); - rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, &apReqs[i], 1); - AssertRC(rc); - } - else - { - if (pTask->fBounceBuffer) - { - if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ) - memcpy(pTask->DataSeg.pvSeg, - ((uint8_t *)pTask->pvBounceBuffer) + pTask->uBounceBufOffset, - pTask->DataSeg.cbSeg); - - RTMemPageFree(pTask->pvBounceBuffer); - } - - /* Put the entry on the free array */ - pAioMgr->pahReqsFree[pAioMgr->iFreeEntryNext] = apReqs[i]; - pAioMgr->iFreeEntryNext = (pAioMgr->iFreeEntryNext + 1) % pAioMgr->cReqEntries; - - pAioMgr->cRequestsActive--; - pEndpoint->AioMgr.cRequestsActive--; - pEndpoint->AioMgr.cReqsProcessed++; - - /* Free the lock and process pending tasks if neccessary */ - pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock); - - /* Call completion callback */ - pTask->pfnCompleted(pTask, pTask->pvUser); - pdmacFileTaskFree(pEndpoint, pTask); - - /* - * If there is no request left on the endpoint but a flush request is set - * it completed now and we notify the owner. - * Furthermore we look for new requests and continue. - */ - if (!pEndpoint->AioMgr.cRequestsActive && pEndpoint->pFlushReq) - { - /* Call completion callback */ - pTask = pEndpoint->pFlushReq; - pEndpoint->pFlushReq = NULL; - - AssertMsg(pTask->pEndpoint == pEndpoint, ("Endpoint of the flush request does not match assigned one\n")); - - pTask->pfnCompleted(pTask, pTask->pvUser); - pdmacFileTaskFree(pEndpoint, pTask); - } - else if (RT_UNLIKELY(!pEndpoint->AioMgr.cRequestsActive && pEndpoint->AioMgr.fMoving)) - { - /* If the endpoint is about to be migrated do it now. */ - bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint); - Assert(!fReqsPending); - - rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint); - AssertRC(rc); - } - } - } /* request completed successfully */ - } /* for every completed request */ - - /* Check for an external blocking event before we go to sleep again. */ - if (pAioMgr->fBlockingEventPending) - { - rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr); - CHECK_RC(pAioMgr, rc); - } - - /* Update load statistics. */ - uint64_t uMillisCurr = RTTimeMilliTS(); - if (uMillisCurr > uMillisEnd) - { - PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointCurr = pAioMgr->pEndpointsHead; - - /* Calculate timespan. */ - uMillisCurr -= uMillisEnd; - - while (pEndpointCurr) - { - pEndpointCurr->AioMgr.cReqsPerSec = pEndpointCurr->AioMgr.cReqsProcessed / (uMillisCurr + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD); - pEndpointCurr->AioMgr.cReqsProcessed = 0; - pEndpointCurr = pEndpointCurr->AioMgr.pEndpointNext; - } - - /* Set new update interval */ - uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD; + RTThreadYield(); }