Index: /trunk/src/VBox/VMM/PDMAsyncCompletionFile.cpp =================================================================== --- /trunk/src/VBox/VMM/PDMAsyncCompletionFile.cpp (revision 22756) +++ /trunk/src/VBox/VMM/PDMAsyncCompletionFile.cpp (revision 22757) @@ -236,5 +236,5 @@ } -static int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint) +int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint) { int rc; @@ -343,5 +343,6 @@ || (enmTransfer == PDMACTASKFILETRANSFER_WRITE)); - pTaskFile->cbTransferLeft = cbTransfer; + ASMAtomicWriteS32(&pTaskFile->cbTransferLeft, cbTransfer); + ASMAtomicWriteBool(&pTaskFile->fCompleted, false); for (unsigned i = 0; i < cSegments; i++) @@ -366,4 +367,8 @@ AssertMsg(!cbTransfer, ("Incomplete transfer %u bytes left\n", cbTransfer)); + if (ASMAtomicReadS32(&pTaskFile->cbTransferLeft) == 0 + && !ASMAtomicXchgBool(&pTaskFile->fCompleted, true)) + pdmR3AsyncCompletionCompleteTask(pTask); + return VINF_SUCCESS; } @@ -376,5 +381,5 @@ * @param ppAioMgr Where to store the pointer to the new async I/O manager on success. */ -static int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr) +int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr) { int rc = VINF_SUCCESS; @@ -577,4 +582,6 @@ } + pEpFile->cbFile = cbSize; + RTFileClose(File); } @@ -737,5 +744,7 @@ PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint; - return RTFileGetSize(pEpFile->File, pcbSize); + *pcbSize = ASMAtomicReadU64(&pEpFile->cbFile); + + return VINF_SUCCESS; } Index: /trunk/src/VBox/VMM/PDMAsyncCompletionFileCache.cpp =================================================================== --- /trunk/src/VBox/VMM/PDMAsyncCompletionFileCache.cpp (revision 22756) +++ /trunk/src/VBox/VMM/PDMAsyncCompletionFileCache.cpp (revision 22757) @@ -52,8 +52,30 @@ #include "PDMAsyncCompletionFileInternal.h" +#ifdef VBOX_STRICT +# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \ + do \ + { \ + AssertMsg(RTCritSectIsOwner(&pCache->CritSect), \ + ("Thread does not own critical section\n"));\ + } while(0); +#else +# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0); +#endif + /******************************************************************************* * Internal Functions * *******************************************************************************/ static void pdmacFileCacheTaskCompleted(PPDMACTASKFILE pTask, void *pvUser); + +DECLINLINE(void) pdmacFileEpCacheEntryRelease(PPDMACFILECACHEENTRY pEntry) +{ + AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n")); + ASMAtomicDecU32(&pEntry->cRefs); +} + +DECLINLINE(void) pdmacFileEpCacheEntryRef(PPDMACFILECACHEENTRY pEntry) +{ + ASMAtomicIncU32(&pEntry->cRefs); +} /** @@ -217,4 +239,6 @@ size_t cbEvicted = 0; + PDMACFILECACHE_IS_CRITSECT_OWNER(pCache); + AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n")); AssertMsg( !pGhostListDst @@ -233,30 +257,41 @@ /* We can't evict pages which are currently in progress */ - if (!(pCurr->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS)) + if (!(pCurr->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS) + && (ASMAtomicReadU32(&pCurr->cRefs) == 0)) { - LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData)); - if (pCurr->pbData) + /* Ok eviction candidate. Grab the endpoint semaphore and check again + * because somebody else might have raced us. */ + PPDMACFILEENDPOINTCACHE pEndpointCache = &pCurr->pEndpoint->DataCache; + RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT); + + if (!(pCurr->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS) + && (ASMAtomicReadU32(&pCurr->cRefs) == 0)) { - RTMemPageFree(pCurr->pbData); - pCurr->pbData = NULL; + LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData)); + if (pCurr->pbData) + { + RTMemPageFree(pCurr->pbData); + pCurr->pbData = NULL; + } + + cbEvicted += pCurr->cbData; + + if (pGhostListDst) + { + pdmacFileCacheEntryAddToList(pGhostListDst, pCurr); + } + else + { + /* Delete the entry from the AVL tree it is assigned to. */ + STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache); + RTAvlrFileOffsetRemove(pCurr->pEndpoint->DataCache.pTree, pCurr->Core.Key); + STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache); + + pdmacFileCacheEntryRemoveFromList(pCurr); + pCache->cbCached -= pCurr->cbData; + RTMemFree(pCurr); + } } - - cbEvicted += pCurr->cbData; - - if (pGhostListDst) - { - pdmacFileCacheEntryAddToList(pGhostListDst, pCurr); - } - else - { - /* Delete the entry from the AVL tree it is assigned to. */ - STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache); - RTAvlrFileOffsetRemove(pCurr->pEndpoint->DataCache.pTree, pCurr->Core.Key); - STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache); - - pdmacFileCacheEntryRemoveFromList(pCurr); - pCache->cbCached -= pCurr->cbData; - RTMemFree(pCurr); - } + RTSemRWReleaseWrite(pEndpointCache->SemRWEntries); } else @@ -269,4 +304,6 @@ static size_t pdmacFileCacheReplace(PPDMACFILECACHEGLOBAL pCache, size_t cbData, PPDMACFILELRULIST pEntryList) { + PDMACFILECACHE_IS_CRITSECT_OWNER(pCache); + if ( (pCache->LruRecentlyUsed.cbCached) && ( (pCache->LruRecentlyUsed.cbCached > pCache->uAdaptVal) @@ -298,4 +335,6 @@ { size_t cbRemoved = ~0; + + PDMACFILECACHE_IS_CRITSECT_OWNER(pCache); if ((pCache->LruRecentlyUsed.cbCached + pCache->LruRecentlyGhost.cbCached) >= pCache->cbMax) @@ -346,4 +385,6 @@ int32_t uUpdateVal = 0; + PDMACFILECACHE_IS_CRITSECT_OWNER(pCache); + /* Update parameters */ if (pEntry->pList == &pCache->LruRecentlyGhost) @@ -436,7 +477,11 @@ PPDMACFILECACHEENTRY pEntry = (PPDMACFILECACHEENTRY)pvUser; PPDMACFILECACHEGLOBAL pCache = pEntry->pCache; - - RTCritSectEnter(&pCache->CritSect); - + PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pEntry->pEndpoint; + + /* Reference the entry now as we are clearing the I/O in progres flag + * which protects the entry till now. */ + pdmacFileEpCacheEntryRef(pEntry); + + RTSemRWRequestWrite(pEndpoint->DataCache.SemRWEntries, RT_INDEFINITE_WAIT); pEntry->fFlags &= ~PDMACFILECACHE_ENTRY_IO_IN_PROGRESS; @@ -503,5 +548,8 @@ pdmacFileCacheWriteToEndpoint(pEntry); - RTCritSectLeave(&pCache->CritSect); + RTSemRWReleaseWrite(pEndpoint->DataCache.SemRWEntries); + + /* Dereference so that it isn't protected anymore except we issued anyother write for it. */ + pdmacFileEpCacheEntryRelease(pEntry); } @@ -648,8 +696,18 @@ PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache; - pEndpointCache->pTree = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE)); pEndpointCache->pCache = &pClassFile->Cache; - return VINF_SUCCESS; + int rc = RTSemRWCreate(&pEndpointCache->SemRWEntries); + if (RT_SUCCESS(rc)) + { + pEndpointCache->pTree = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE)); + if (!pEndpointCache->pTree) + { + rc = VERR_NO_MEMORY; + RTSemRWDestroy(pEndpointCache->SemRWEntries); + } + } + + return rc; } @@ -690,7 +748,59 @@ /* Make sure nobody is accessing the cache while we delete the tree. */ + RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT); RTCritSectEnter(&pCache->CritSect); RTAvlrFileOffsetDestroy(pEndpointCache->pTree, pdmacFileEpCacheEntryDestroy, pCache); RTCritSectLeave(&pCache->CritSect); + RTSemRWReleaseWrite(pEndpointCache->SemRWEntries); + + RTSemRWDestroy(pEndpointCache->SemRWEntries); +} + +static PPDMACFILECACHEENTRY pdmacFileEpCacheGetCacheEntryByOffset(PPDMACFILEENDPOINTCACHE pEndpointCache, RTFOFF off) +{ + PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache; + PPDMACFILECACHEENTRY pEntry = NULL; + + STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache); + + RTSemRWRequestRead(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT); + pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetRangeGet(pEndpointCache->pTree, off); + if (pEntry) + pdmacFileEpCacheEntryRef(pEntry); + RTSemRWReleaseRead(pEndpointCache->SemRWEntries); + + STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache); + + return pEntry; +} + +static PPDMACFILECACHEENTRY pdmacFileEpCacheGetCacheBestFitEntryByOffset(PPDMACFILEENDPOINTCACHE pEndpointCache, RTFOFF off) +{ + PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache; + PPDMACFILECACHEENTRY pEntry = NULL; + + STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache); + + RTSemRWRequestRead(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT); + pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetGetBestFit(pEndpointCache->pTree, off, true); + if (pEntry) + pdmacFileEpCacheEntryRef(pEntry); + RTSemRWReleaseRead(pEndpointCache->SemRWEntries); + + STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache); + + return pEntry; +} + +static void pdmacFileEpCacheInsertEntry(PPDMACFILEENDPOINTCACHE pEndpointCache, PPDMACFILECACHEENTRY pEntry) +{ + PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache; + + STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache); + RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT); + bool fInserted = RTAvlrFileOffsetInsert(pEndpointCache->pTree, &pEntry->Core); + AssertMsg(fInserted, ("Node was not inserted into tree\n")); + STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache); + RTSemRWReleaseWrite(pEndpointCache->SemRWEntries); } @@ -741,6 +851,4 @@ ASMAtomicWriteBool(&pTask->fCompleted, true); - RTCritSectEnter(&pCache->CritSect); - int iSegCurr = 0; uint8_t *pbSegBuf = (uint8_t *)paSegments[iSegCurr].pvSeg; @@ -751,7 +859,5 @@ size_t cbToRead; - STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache); - pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetRangeGet(pEndpointCache->pTree, off); - STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache); + pEntry = pdmacFileEpCacheGetCacheEntryByOffset(pEndpointCache, off); /* @@ -793,23 +899,50 @@ && !(pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY)) { - /* Entry didn't completed yet. Append to the list */ - while (cbToRead) + RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT); + /* Check again. The completion callback might have raced us. */ + + if ( (pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS) + && !(pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY)) { - PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG)); - - pSeg->pTask = pTask; - pSeg->uBufOffset = OffDiff; - pSeg->cbTransfer = RT_MIN(cbToRead, cbSegLeft); - pSeg->pvBuf = pbSegBuf; - pSeg->fWrite = false; - - ADVANCE_SEGMENT_BUFFER(pSeg->cbTransfer); - - pSeg->pNext = pEntry->pHead; - pEntry->pHead = pSeg; - - off += pSeg->cbTransfer; - cbToRead -= pSeg->cbTransfer; - OffDiff += pSeg->cbTransfer; + /* Entry didn't completed yet. Append to the list */ + while (cbToRead) + { + PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG)); + + pSeg->pTask = pTask; + pSeg->uBufOffset = OffDiff; + pSeg->cbTransfer = RT_MIN(cbToRead, cbSegLeft); + pSeg->pvBuf = pbSegBuf; + pSeg->fWrite = false; + + ADVANCE_SEGMENT_BUFFER(pSeg->cbTransfer); + + pSeg->pNext = pEntry->pHead; + pEntry->pHead = pSeg; + + off += pSeg->cbTransfer; + cbToRead -= pSeg->cbTransfer; + OffDiff += pSeg->cbTransfer; + } + RTSemRWReleaseWrite(pEndpointCache->SemRWEntries); + } + else + { + RTSemRWReleaseWrite(pEndpointCache->SemRWEntries); + + /* Read as much as we can from the entry. */ + while (cbToRead) + { + size_t cbCopy = RT_MIN(cbSegLeft, cbToRead); + + memcpy(pbSegBuf, pEntry->pbData + OffDiff, cbCopy); + + ADVANCE_SEGMENT_BUFFER(cbCopy); + + cbToRead -= cbCopy; + off += cbCopy; + OffDiff += cbCopy; + ASMAtomicSubS32(&pTask->cbTransferLeft, cbCopy); + } } } @@ -833,8 +966,11 @@ /* Move this entry to the top position */ + RTCritSectEnter(&pCache->CritSect); pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry); + RTCritSectLeave(&pCache->CritSect); } else { + RTCritSectEnter(&pCache->CritSect); pdmacFileCacheUpdate(pCache, pEntry); pdmacFileCacheReplace(pCache, pEntry->cbData, pEntry->pList); @@ -842,4 +978,5 @@ /* Move the entry to T2 and fetch it to the cache. */ pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry); + RTCritSectLeave(&pCache->CritSect); pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData); @@ -871,11 +1008,10 @@ pdmacFileCacheReadFromEndpoint(pEntry); } + pdmacFileEpCacheEntryRelease(pEntry); } else { /* No entry found for this offset. Get best fit entry and fetch the data to the cache. */ - STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache); - PPDMACFILECACHEENTRY pEntryBestFit = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetGetBestFit(pEndpointCache->pTree, off, true); - STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache); + PPDMACFILECACHEENTRY pEntryBestFit = pdmacFileEpCacheGetCacheBestFitEntryByOffset(pEndpointCache, off); LogFlow(("%sbest fit entry for off=%RTfoff (BestFit=%RTfoff BestFitEnd=%RTfoff BestFitSize=%u)\n", @@ -887,5 +1023,8 @@ if (pEntryBestFit && ((off + (RTFOFF)cbRead) > pEntryBestFit->Core.Key)) + { cbToRead = pEntryBestFit->Core.Key - off; + pdmacFileEpCacheEntryRelease(pEntryBestFit); + } else cbToRead = cbRead; @@ -898,5 +1037,7 @@ STAM_COUNTER_INC(&pCache->cPartialHits); + RTCritSectEnter(&pCache->CritSect); size_t cbRemoved = pdmacFileCacheEvict(pCache, cbToRead); + RTCritSectLeave(&pCache->CritSect); if (cbRemoved >= cbToRead) @@ -911,4 +1052,5 @@ pEntryNew->pCache = pCache; pEntryNew->fFlags = 0; + pEntryNew->cRefs = 1; /* We are using it now. */ pEntryNew->pList = NULL; pEntryNew->cbData = cbToRead; @@ -916,11 +1058,10 @@ pEntryNew->pbData = (uint8_t *)RTMemPageAlloc(cbToRead); AssertPtr(pEntryNew->pbData); + + RTCritSectEnter(&pCache->CritSect); pdmacFileCacheEntryAddToList(&pCache->LruRecentlyUsed, pEntryNew); - - STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache); - bool fInserted = RTAvlrFileOffsetInsert(pEndpoint->DataCache.pTree, &pEntryNew->Core); - AssertMsg(fInserted, ("Node was not inserted into tree\n")); - STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache); - + RTCritSectLeave(&pCache->CritSect); + + pdmacFileEpCacheInsertEntry(pEndpointCache, pEntryNew); uint32_t uBufOffset = 0; @@ -947,4 +1088,5 @@ pdmacFileCacheReadFromEndpoint(pEntryNew); + pdmacFileEpCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */ } else @@ -987,6 +1129,4 @@ pdmR3AsyncCompletionCompleteTask(&pTask->Core); - RTCritSectLeave(&pCache->CritSect); - return rc; } @@ -1019,6 +1159,4 @@ ASMAtomicWriteBool(&pTask->fCompleted, true); - RTCritSectEnter(&pCache->CritSect); - int iSegCurr = 0; uint8_t *pbSegBuf = (uint8_t *)paSegments[iSegCurr].pvSeg; @@ -1029,7 +1167,5 @@ size_t cbToWrite; - STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache); - pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetRangeGet(pEndpointCache->pTree, off); - STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache); + pEntry = pdmacFileEpCacheGetCacheEntryByOffset(pEndpointCache, off); if (pEntry) @@ -1060,26 +1196,58 @@ if (pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY) { - AssertMsg(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS, - ("Entry is dirty but not in progress\n")); - - /* The data isn't written to the file yet */ - while (cbToWrite) + RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT); + /* Check again. The completion callback might have raced us. */ + + if (pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY) { - PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG)); - - pSeg->pTask = pTask; - pSeg->uBufOffset = OffDiff; - pSeg->cbTransfer = RT_MIN(cbToWrite, cbSegLeft); - pSeg->pvBuf = pbSegBuf; - pSeg->fWrite = true; - - ADVANCE_SEGMENT_BUFFER(pSeg->cbTransfer); - - pSeg->pNext = pEntry->pHead; - pEntry->pHead = pSeg; - - off += pSeg->cbTransfer; - OffDiff += pSeg->cbTransfer; - cbToWrite -= pSeg->cbTransfer; + AssertMsg(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS, + ("Entry is dirty but not in progress\n")); + + /* The data isn't written to the file yet */ + while (cbToWrite) + { + PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG)); + + pSeg->pTask = pTask; + pSeg->uBufOffset = OffDiff; + pSeg->cbTransfer = RT_MIN(cbToWrite, cbSegLeft); + pSeg->pvBuf = pbSegBuf; + pSeg->fWrite = true; + + ADVANCE_SEGMENT_BUFFER(pSeg->cbTransfer); + + pSeg->pNext = pEntry->pHead; + pEntry->pHead = pSeg; + + off += pSeg->cbTransfer; + OffDiff += pSeg->cbTransfer; + cbToWrite -= pSeg->cbTransfer; + } + RTSemRWReleaseWrite(pEndpointCache->SemRWEntries); + } + else + { + RTSemRWReleaseWrite(pEndpointCache->SemRWEntries); + + AssertMsg(!(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS), + ("Entry is not dirty but in progress\n")); + + /* Write as much as we can into the entry and update the file. */ + while (cbToWrite) + { + size_t cbCopy = RT_MIN(cbSegLeft, cbToWrite); + + memcpy(pEntry->pbData + OffDiff, pbSegBuf, cbCopy); + + ADVANCE_SEGMENT_BUFFER(cbCopy); + + cbToWrite-= cbCopy; + off += cbCopy; + OffDiff += cbCopy; + ASMAtomicSubS32(&pTask->cbTransferLeft, cbCopy); + } + + pEntry->fFlags |= PDMACFILECACHE_ENTRY_IS_DIRTY; + pdmacFileCacheWriteToEndpoint(pEntry); } } @@ -1109,8 +1277,11 @@ /* Move this entry to the top position */ + RTCritSectEnter(&pCache->CritSect); pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry); + RTCritSectLeave(&pCache->CritSect); } else { + RTCritSectEnter(&pCache->CritSect); pdmacFileCacheUpdate(pCache, pEntry); pdmacFileCacheReplace(pCache, pEntry->cbData, pEntry->pList); @@ -1118,4 +1289,5 @@ /* Move the entry to T2 and fetch it to the cache. */ pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry); + RTCritSectLeave(&pCache->CritSect); pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData); @@ -1148,4 +1320,7 @@ pdmacFileCacheReadFromEndpoint(pEntry); } + + /* Release the reference. If it is still needed the I/O in progress flag should protect it now. */ + pdmacFileEpCacheEntryRelease(pEntry); } else @@ -1154,7 +1329,5 @@ * No entry found. Write directly into file. */ - STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache); - PPDMACFILECACHEENTRY pEntryBestFit = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetGetBestFit(pEndpointCache->pTree, off, true); - STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache); + PPDMACFILECACHEENTRY pEntryBestFit = pdmacFileEpCacheGetCacheBestFitEntryByOffset(pEndpointCache, off); LogFlow(("%sbest fit entry for off=%RTfoff (BestFit=%RTfoff BestFitEnd=%RTfoff BestFitSize=%u)\n", @@ -1166,5 +1339,8 @@ if (pEntryBestFit && ((off + (RTFOFF)cbWrite) > pEntryBestFit->Core.Key)) + { cbToWrite = pEntryBestFit->Core.Key - off; + pdmacFileEpCacheEntryRelease(pEntryBestFit); + } else cbToWrite = cbWrite; @@ -1202,6 +1378,4 @@ pdmR3AsyncCompletionCompleteTask(&pTask->Core); - RTCritSectLeave(&pCache->CritSect); - return VINF_SUCCESS; } Index: /trunk/src/VBox/VMM/PDMAsyncCompletionFileFailsafe.cpp =================================================================== --- /trunk/src/VBox/VMM/PDMAsyncCompletionFileFailsafe.cpp (revision 22756) +++ /trunk/src/VBox/VMM/PDMAsyncCompletionFileFailsafe.cpp (revision 22757) @@ -47,4 +47,10 @@ if (pCurr->enmTransferType == PDMACTASKFILETRANSFER_READ) { + if (RT_UNLIKELY((pCurr->Off + pCurr->DataSeg.cbSeg) > pEndpoint->cbFile)) + { + ASMAtomicWriteU64(&pEndpoint->cbFile, pCurr->Off + pCurr->DataSeg.cbSeg); + RTFileSetSize(pEndpoint->File, pCurr->Off + pCurr->DataSeg.cbSeg); + } + rc = RTFileReadAt(pEndpoint->File, pCurr->Off, pCurr->DataSeg.pvSeg, Index: /trunk/src/VBox/VMM/PDMAsyncCompletionFileInternal.h =================================================================== --- /trunk/src/VBox/VMM/PDMAsyncCompletionFileInternal.h (revision 22756) +++ /trunk/src/VBox/VMM/PDMAsyncCompletionFileInternal.h (revision 22757) @@ -134,4 +134,6 @@ /** List of endpoints assigned to this manager. */ R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointsHead; + /** Number of endpoints assigned to the manager. */ + unsigned cEndpoints; /** Number of requests active currently. */ unsigned cRequestsActive; @@ -221,4 +223,6 @@ /** Flags for this entry. Combinations of PDMACFILECACHE_* #defines */ uint32_t fFlags; + /** Reference counter. Prevents eviction of the entry if > 0. */ + volatile uint32_t cRefs; /** Size of the entry. */ size_t cbData; @@ -298,6 +302,6 @@ /** AVL tree managing cache entries. */ PAVLRFOFFTREE pTree; - /** Critical section protecting the tree. */ - RTCRITSECT CritSect; + /** R/W semaphore protecting cached entries for this endpoint. */ + RTSEMRW SemRWEntries; /** Pointer to the gobal cache data */ PPDMACFILECACHEGLOBAL pCache; @@ -383,5 +387,5 @@ /** Size of the underlying file. * Updated while data is appended. */ - uint64_t cbFile; + volatile uint64_t cbFile; /** Flag whether caching is enabled for this file. */ bool fCaching; @@ -444,4 +448,8 @@ /** Current number of processed requests for the current update period. */ unsigned cReqsProcessed; + /** Flag whether the endpoint is about to be moved to another manager. */ + bool fMoving; + /** Destination I/O manager. */ + PPDMACEPFILEMGR pAioMgrDst; } AioMgr; } PDMASYNCCOMPLETIONENDPOINTFILE; @@ -514,4 +522,8 @@ void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr); +int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr); + +int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint); + PPDMACTASKFILE pdmacFileEpGetNewTasks(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint); PPDMACTASKFILE pdmacFileTaskAlloc(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint); Index: /trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp =================================================================== --- /trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp (revision 22756) +++ /trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp (revision 22757) @@ -22,4 +22,5 @@ #define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION #include +#include #include #include @@ -75,4 +76,190 @@ RTMemFree(pAioMgr->pahReqsFree); +} + +/** + * Sorts the endpoint list with insertion sort. + */ +static void pdmacFileAioMgrNormalEndpointsSortByLoad(PPDMACEPFILEMGR pAioMgr) +{ + PPDMASYNCCOMPLETIONENDPOINTFILE pEpPrev, pEpCurr, pEpNextToSort; + + pEpPrev = pAioMgr->pEndpointsHead; + pEpCurr = pEpPrev->AioMgr.pEndpointNext; + + while (pEpCurr) + { + /* Remember the next element to sort because the list might change. */ + pEpNextToSort = pEpCurr->AioMgr.pEndpointNext; + + /* Unlink the current element from the list. */ + PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev; + PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext; + + if (pPrev) + pPrev->AioMgr.pEndpointNext = pNext; + else + pAioMgr->pEndpointsHead = pNext; + + if (pNext) + pNext->AioMgr.pEndpointPrev = pPrev; + + /* Go back until we reached the place to insert the current endpoint into. */ + while (pEpPrev && (pEpPrev->AioMgr.cReqsPerSec < pEpCurr->AioMgr.cReqsPerSec)) + pEpPrev = pEpPrev->AioMgr.pEndpointPrev; + + /* Link the endpoint into the list. */ + if (pEpPrev) + pNext = pEpPrev->AioMgr.pEndpointNext; + else + pNext = pAioMgr->pEndpointsHead; + + pEpCurr->AioMgr.pEndpointNext = pNext; + pEpCurr->AioMgr.pEndpointPrev = pEpPrev; + pNext->AioMgr.pEndpointPrev = pEpCurr; + if (pEpPrev) + pEpPrev->AioMgr.pEndpointNext = pEpCurr; + else + pAioMgr->pEndpointsHead = pEpCurr; + + pEpCurr = pEpNextToSort; + } + +#ifdef DEBUG + /* Validate sorting alogrithm */ + unsigned cEndpoints = 0; + pEpCurr = pAioMgr->pEndpointsHead; + + AssertMsg(pEpCurr, ("No endpoint in the list?\n")); + AssertMsg(!pEpCurr->AioMgr.pEndpointPrev, ("First element in the list points to previous element\n")); + + while (pEpCurr) + { + cEndpoints++; + + PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext; + PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev; + + Assert(!pNext || pNext->AioMgr.cReqsPerSec <= pEpCurr->AioMgr.cReqsPerSec); + Assert(!pPrev || pPrev->AioMgr.cReqsPerSec >= pEpCurr->AioMgr.cReqsPerSec); + + pEpCurr = pNext; + } + + AssertMsg(cEndpoints == pAioMgr->cEndpoints, ("Endpoints lost during sort!\n")); + +#endif +} + +/** + * Removes an endpoint from the currently assigned manager. + * + * @returns TRUE if there are still requests pending on the current manager for this endpoint. + * FALSE otherwise. + * @param pEndpointRemove The endpoint to remove. + */ +static bool pdmacFileAioMgrNormalRemoveEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove) +{ + PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev; + PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext; + PPDMACEPFILEMGR pAioMgr = pEndpointRemove->pAioMgr; + + pAioMgr->cEndpoints--; + + if (pPrev) + pPrev->AioMgr.pEndpointNext = pNext; + else + pAioMgr->pEndpointsHead = pNext; + + if (pNext) + pNext->AioMgr.pEndpointPrev = pPrev; + + /* Make sure that there is no request pending on this manager for the endpoint. */ + if (!pEndpointRemove->AioMgr.cRequestsActive) + { + Assert(!pEndpointRemove->pFlushReq); + + /* Reopen the file so that the new endpoint can reassociate with the file */ + RTFileClose(pEndpointRemove->File); + int rc = RTFileOpen(&pEndpointRemove->File, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags); + AssertRC(rc); + return false; + } + + return true; +} + +/** + * Creates a new I/O manager and spreads the I/O load of the endpoints + * between the given I/O manager and the new one. + * + * @returns nothing. + * @param pAioMgr The I/O manager with high I/O load. + */ +static void pdmacFileAioMgrNormalBalanceLoad(PPDMACEPFILEMGR pAioMgr) +{ + PPDMACEPFILEMGR pAioMgrNew = NULL; + int rc = VINF_SUCCESS; + + /* Splitting can't be done with only one open endpoint. */ + if (pAioMgr->cEndpoints > 1) + { + rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass, + &pAioMgrNew); + if (RT_SUCCESS(rc)) + { + /* We will sort the list by request count per second. */ + pdmacFileAioMgrNormalEndpointsSortByLoad(pAioMgr); + + /* Now move some endpoints to the new manager. */ + unsigned cReqsHere = pAioMgr->pEndpointsHead->AioMgr.cReqsPerSec; + unsigned cReqsOther = 0; + PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead->AioMgr.pEndpointNext; + + while (pCurr) + { + if (cReqsHere <= cReqsOther) + { + /* + * The other manager has more requests to handle now. + * We will keep the current endpoint. + */ + Log(("Keeping endpoint %#p{%s} with %u reqs/s\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec)); + cReqsHere += pCurr->AioMgr.cReqsPerSec; + pCurr = pCurr->AioMgr.pEndpointNext; + } + else + { + /* Move to other endpoint. */ + Log(("Moving endpoint %#p{%s} with %u reqs/s to other manager\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec)); + cReqsOther += pCurr->AioMgr.cReqsPerSec; + + PPDMASYNCCOMPLETIONENDPOINTFILE pMove = pCurr; + + pCurr = pCurr->AioMgr.pEndpointNext; + + bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pMove); + + if (fReqsPending) + { + pMove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING; + pMove->AioMgr.fMoving = true; + pMove->AioMgr.pAioMgrDst = pAioMgrNew; + } + else + { + pMove->AioMgr.fMoving = false; + pMove->AioMgr.pAioMgrDst = NULL; + pdmacFileAioMgrAddEndpoint(pAioMgrNew, pMove); + } + } + } + } + else + { + /* Don't process further but leave a log entry about reduced performance. */ + LogRel(("AIOMgr: Could not create new I/O manager (rc=%Rrc). Expect reduced performance\n", rc)); + } + } } @@ -106,4 +293,5 @@ RTFILEAIOREQ apReqs[20]; unsigned cRequests = 0; + unsigned cMaxRequests = PDMACEPFILEMGR_REQS_MAX - pAioMgr->cRequestsActive; int rc = VINF_SUCCESS; PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass; @@ -113,5 +301,5 @@ /* Go through the list and queue the requests until we get a flush request */ - while (pTaskHead && !pEndpoint->pFlushReq) + while (pTaskHead && !pEndpoint->pFlushReq && (cMaxRequests > 0)) { PPDMACTASKFILE pCurr = pTaskHead; @@ -205,6 +393,15 @@ if (pCurr->enmTransferType == PDMACTASKFILETRANSFER_WRITE) + { + /* Grow the file if needed. */ + if (RT_UNLIKELY((pCurr->Off + pCurr->DataSeg.cbSeg) > pEndpoint->cbFile)) + { + ASMAtomicWriteU64(&pEndpoint->cbFile, pCurr->Off + pCurr->DataSeg.cbSeg); + RTFileSetSize(pEndpoint->File, pCurr->Off + pCurr->DataSeg.cbSeg); + } + rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File, pCurr->Off, pvBuf, pCurr->DataSeg.cbSeg, pCurr); + } else rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File, @@ -214,4 +411,5 @@ apReqs[cRequests] = hReq; pEndpoint->AioMgr.cReqsProcessed++; + cMaxRequests--; cRequests++; if (cRequests == RT_ELEMENTS(apReqs)) @@ -238,10 +436,34 @@ pAioMgr->cRequestsActive += cRequests; rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, apReqs, cRequests); - if (RT_FAILURE(rc)) - { - /* Not enough ressources on this context anymore. */ - /* @todo implement */ - AssertMsgFailed(("Implement\n")); - } + AssertMsgReturn(RT_SUCCESS(rc), ("Could not submit %u requests %Rrc\n", cRequests, rc), rc); + } + + if (RT_UNLIKELY(!cMaxRequests && pTaskHead && !pEndpoint->pFlushReq)) + { + /* + * The I/O manager has no room left for more requests + * but there are still requests to process. + * Create a new I/O manager and let it handle some endpoints. + */ + + /* Add the rest of the tasks to the pending list first */ + if (!pEndpoint->AioMgr.pReqsPendingHead) + { + Assert(!pEndpoint->AioMgr.pReqsPendingTail); + pEndpoint->AioMgr.pReqsPendingHead = pTaskHead; + } + else + { + Assert(pEndpoint->AioMgr.pReqsPendingTail); + pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead; + } + + /* Update the tail. */ + while (pTaskHead->pNext) + pTaskHead = pTaskHead->pNext; + + pEndpoint->AioMgr.pReqsPendingTail = pTaskHead; + + pdmacFileAioMgrNormalBalanceLoad(pAioMgr); } @@ -322,4 +544,5 @@ rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pEndpointNew->File); fNotifyWaiter = true; + pAioMgr->cEndpoints++; break; } @@ -329,27 +552,6 @@ AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n")); - PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev; - PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext; - pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING; - - if (pPrev) - pPrev->AioMgr.pEndpointNext = pNext; - else - pAioMgr->pEndpointsHead = pNext; - - if (pNext) - pNext->AioMgr.pEndpointPrev = pPrev; - - /* Make sure that there is no request pending on this manager for the endpoint. */ - if (!pEndpointRemove->AioMgr.cRequestsActive) - { - Assert(!pEndpointRemove->pFlushReq); - - /* Reopen the file so that the new endpoint can reassociate with the file */ - RTFileClose(pEndpointRemove->File); - rc = RTFileOpen(&pEndpointRemove->File, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags); - AssertRC(rc); - } + fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointRemove); break; } @@ -364,26 +566,5 @@ pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING; - - PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointClose->AioMgr.pEndpointPrev; - PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointClose->AioMgr.pEndpointNext; - - if (pPrev) - pPrev->AioMgr.pEndpointNext = pNext; - else - pAioMgr->pEndpointsHead = pNext; - - if (pNext) - pNext->AioMgr.pEndpointPrev = pPrev; - - if (!pEndpointClose->AioMgr.cRequestsActive) - { - Assert(!pEndpointClose->pFlushReq); - - /* Reopen the file to deassociate it from the endpoint. */ - RTFileClose(pEndpointClose->File); - rc = RTFileOpen(&pEndpointClose->File, pEndpointClose->Core.pszUri, pEndpointClose->fFlags); - AssertRC(rc); - fNotifyWaiter = true; - } + fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointClose); break; } @@ -483,8 +664,17 @@ RTFILEAIOREQ apReqs[20]; uint32_t cReqsCompleted = 0; - - rc = RTFileAioCtxWait(pAioMgr->hAioCtx, 1, RT_INDEFINITE_WAIT, apReqs, + size_t cReqsWait; + + if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs)) + cReqsWait = RT_ELEMENTS(apReqs); + else + cReqsWait = pAioMgr->cRequestsActive; + + rc = RTFileAioCtxWait(pAioMgr->hAioCtx, + cReqsWait, + RT_INDEFINITE_WAIT, apReqs, RT_ELEMENTS(apReqs), &cReqsCompleted); - CHECK_RC(pAioMgr, rc); + if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED)) + CHECK_RC(pAioMgr, rc); for (uint32_t i = 0; i < cReqsCompleted; i++) @@ -547,10 +737,23 @@ else if (!pEndpoint->AioMgr.cRequestsActive) { - Assert(pAioMgr->fBlockingEventPending); - ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false); - - /* Release the waiting thread. */ - rc = RTSemEventSignal(pAioMgr->EventSemBlock); + /* Reopen the file so that the new endpoint can reassociate with the file */ + RTFileClose(pEndpoint->File); + rc = RTFileOpen(&pEndpoint->File, pEndpoint->Core.pszUri, pEndpoint->fFlags); AssertRC(rc); + + if (pEndpoint->AioMgr.fMoving) + { + pEndpoint->AioMgr.fMoving = false; + pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint); + } + else + { + Assert(pAioMgr->fBlockingEventPending); + ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false); + + /* Release the waiting thread. */ + rc = RTSemEventSignal(pAioMgr->EventSemBlock); + AssertRC(rc); + } } }