VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/PDMBlkCache.cpp@ 36568

Last change on this file since 36568 was 36568, checked in by vboxsync, 14 years ago

PDMBlkCache: fixed memory leak

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 90.8 KB
Line 
1/* $Id: PDMBlkCache.cpp 36568 2011-04-06 08:51:17Z vboxsync $ */
2/** @file
3 * PDM Block Cache.
4 */
5
6/*
7 * Copyright (C) 2006-2008 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/** @page pg_pdm_block_cache PDM Block Cache - The I/O cache
19 * This component implements an I/O cache based on the 2Q cache algorithm.
20 */
21
22/*******************************************************************************
23* Header Files *
24*******************************************************************************/
25#define LOG_GROUP LOG_GROUP_PDM_BLK_CACHE
26#include "PDMInternal.h"
27#include <iprt/asm.h>
28#include <iprt/mem.h>
29#include <iprt/path.h>
30#include <iprt/string.h>
31#include <VBox/log.h>
32#include <VBox/vmm/stam.h>
33#include <VBox/vmm/uvm.h>
34#include <VBox/vmm/vm.h>
35
36#include "PDMBlkCacheInternal.h"
37
38#ifdef VBOX_STRICT
39# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
40 do \
41 { \
42 AssertMsg(RTCritSectIsOwner(&Cache->CritSect), \
43 ("Thread does not own critical section\n"));\
44 } while(0)
45
46# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) \
47 do \
48 { \
49 AssertMsg(RTSemRWIsWriteOwner(pEpCache->SemRWEntries), \
50 ("Thread is not exclusive owner of the per endpoint RW semaphore\n")); \
51 } while(0)
52
53# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) \
54 do \
55 { \
56 AssertMsg(RTSemRWIsReadOwner(pEpCache->SemRWEntries), \
57 ("Thread is not read owner of the per endpoint RW semaphore\n")); \
58 } while(0)
59
60#else
61# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0)
62# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) do { } while(0)
63# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) do { } while(0)
64#endif
65
66#define PDM_BLK_CACHE_SAVED_STATE_VERSION 1
67
68/*******************************************************************************
69* Internal Functions *
70*******************************************************************************/
71
72static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
73 uint64_t off, size_t cbData, uint8_t *pbBuffer);
74static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry);
75
76/**
77 * Decrement the reference counter of the given cache entry.
78 *
79 * @returns nothing.
80 * @param pEntry The entry to release.
81 */
82DECLINLINE(void) pdmBlkCacheEntryRelease(PPDMBLKCACHEENTRY pEntry)
83{
84 AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
85 ASMAtomicDecU32(&pEntry->cRefs);
86}
87
88/**
89 * Increment the reference counter of the given cache entry.
90 *
91 * @returns nothing.
92 * @param pEntry The entry to reference.
93 */
94DECLINLINE(void) pdmBlkCacheEntryRef(PPDMBLKCACHEENTRY pEntry)
95{
96 ASMAtomicIncU32(&pEntry->cRefs);
97}
98
99#ifdef DEBUG
100static void pdmBlkCacheValidate(PPDMBLKCACHEGLOBAL pCache)
101{
102 /* Amount of cached data should never exceed the maximum amount. */
103 AssertMsg(pCache->cbCached <= pCache->cbMax,
104 ("Current amount of cached data exceeds maximum\n"));
105
106 /* The amount of cached data in the LRU and FRU list should match cbCached */
107 AssertMsg(pCache->LruRecentlyUsedIn.cbCached + pCache->LruFrequentlyUsed.cbCached == pCache->cbCached,
108 ("Amount of cached data doesn't match\n"));
109
110 AssertMsg(pCache->LruRecentlyUsedOut.cbCached <= pCache->cbRecentlyUsedOutMax,
111 ("Paged out list exceeds maximum\n"));
112}
113#endif
114
115DECLINLINE(void) pdmBlkCacheLockEnter(PPDMBLKCACHEGLOBAL pCache)
116{
117 RTCritSectEnter(&pCache->CritSect);
118#ifdef DEBUG
119 pdmBlkCacheValidate(pCache);
120#endif
121}
122
123DECLINLINE(void) pdmBlkCacheLockLeave(PPDMBLKCACHEGLOBAL pCache)
124{
125#ifdef DEBUG
126 pdmBlkCacheValidate(pCache);
127#endif
128 RTCritSectLeave(&pCache->CritSect);
129}
130
131DECLINLINE(void) pdmBlkCacheSub(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
132{
133 PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
134 pCache->cbCached -= cbAmount;
135}
136
137DECLINLINE(void) pdmBlkCacheAdd(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
138{
139 PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
140 pCache->cbCached += cbAmount;
141}
142
143DECLINLINE(void) pdmBlkCacheListAdd(PPDMBLKLRULIST pList, uint32_t cbAmount)
144{
145 pList->cbCached += cbAmount;
146}
147
148DECLINLINE(void) pdmBlkCacheListSub(PPDMBLKLRULIST pList, uint32_t cbAmount)
149{
150 pList->cbCached -= cbAmount;
151}
152
153#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
154/**
155 * Checks consistency of a LRU list.
156 *
157 * @returns nothing
158 * @param pList The LRU list to check.
159 * @param pNotInList Element which is not allowed to occur in the list.
160 */
161static void pdmBlkCacheCheckList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pNotInList)
162{
163 PPDMBLKCACHEENTRY pCurr = pList->pHead;
164
165 /* Check that there are no double entries and no cycles in the list. */
166 while (pCurr)
167 {
168 PPDMBLKCACHEENTRY pNext = pCurr->pNext;
169
170 while (pNext)
171 {
172 AssertMsg(pCurr != pNext,
173 ("Entry %#p is at least two times in list %#p or there is a cycle in the list\n",
174 pCurr, pList));
175 pNext = pNext->pNext;
176 }
177
178 AssertMsg(pCurr != pNotInList, ("Not allowed entry %#p is in list\n", pCurr));
179
180 if (!pCurr->pNext)
181 AssertMsg(pCurr == pList->pTail, ("End of list reached but last element is not list tail\n"));
182
183 pCurr = pCurr->pNext;
184 }
185}
186#endif
187
188/**
189 * Unlinks a cache entry from the LRU list it is assigned to.
190 *
191 * @returns nothing.
192 * @param pEntry The entry to unlink.
193 */
194static void pdmBlkCacheEntryRemoveFromList(PPDMBLKCACHEENTRY pEntry)
195{
196 PPDMBLKLRULIST pList = pEntry->pList;
197 PPDMBLKCACHEENTRY pPrev, pNext;
198
199 LogFlowFunc((": Deleting entry %#p from list %#p\n", pEntry, pList));
200
201 AssertPtr(pList);
202
203#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
204 pdmBlkCacheCheckList(pList, NULL);
205#endif
206
207 pPrev = pEntry->pPrev;
208 pNext = pEntry->pNext;
209
210 AssertMsg(pEntry != pPrev, ("Entry links to itself as previous element\n"));
211 AssertMsg(pEntry != pNext, ("Entry links to itself as next element\n"));
212
213 if (pPrev)
214 pPrev->pNext = pNext;
215 else
216 {
217 pList->pHead = pNext;
218
219 if (pNext)
220 pNext->pPrev = NULL;
221 }
222
223 if (pNext)
224 pNext->pPrev = pPrev;
225 else
226 {
227 pList->pTail = pPrev;
228
229 if (pPrev)
230 pPrev->pNext = NULL;
231 }
232
233 pEntry->pList = NULL;
234 pEntry->pPrev = NULL;
235 pEntry->pNext = NULL;
236 pdmBlkCacheListSub(pList, pEntry->cbData);
237#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
238 pdmBlkCacheCheckList(pList, pEntry);
239#endif
240}
241
242/**
243 * Adds a cache entry to the given LRU list unlinking it from the currently
244 * assigned list if needed.
245 *
246 * @returns nothing.
247 * @param pList List to the add entry to.
248 * @param pEntry Entry to add.
249 */
250static void pdmBlkCacheEntryAddToList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pEntry)
251{
252 LogFlowFunc((": Adding entry %#p to list %#p\n", pEntry, pList));
253#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
254 pdmBlkCacheCheckList(pList, NULL);
255#endif
256
257 /* Remove from old list if needed */
258 if (pEntry->pList)
259 pdmBlkCacheEntryRemoveFromList(pEntry);
260
261 pEntry->pNext = pList->pHead;
262 if (pList->pHead)
263 pList->pHead->pPrev = pEntry;
264 else
265 {
266 Assert(!pList->pTail);
267 pList->pTail = pEntry;
268 }
269
270 pEntry->pPrev = NULL;
271 pList->pHead = pEntry;
272 pdmBlkCacheListAdd(pList, pEntry->cbData);
273 pEntry->pList = pList;
274#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
275 pdmBlkCacheCheckList(pList, NULL);
276#endif
277}
278
279/**
280 * Destroys a LRU list freeing all entries.
281 *
282 * @returns nothing
283 * @param pList Pointer to the LRU list to destroy.
284 *
285 * @note The caller must own the critical section of the cache.
286 */
287static void pdmBlkCacheDestroyList(PPDMBLKLRULIST pList)
288{
289 while (pList->pHead)
290 {
291 PPDMBLKCACHEENTRY pEntry = pList->pHead;
292
293 pList->pHead = pEntry->pNext;
294
295 AssertMsg(!(pEntry->fFlags & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS | PDMBLKCACHE_ENTRY_IS_DIRTY)),
296 ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
297
298 RTMemPageFree(pEntry->pbData, pEntry->cbData);
299 RTMemFree(pEntry);
300 }
301}
302
303/**
304 * Tries to remove the given amount of bytes from a given list in the cache
305 * moving the entries to one of the given ghosts lists
306 *
307 * @returns Amount of data which could be freed.
308 * @param pCache Pointer to the global cache data.
309 * @param cbData The amount of the data to free.
310 * @param pListSrc The source list to evict data from.
311 * @param pGhostListSrc The ghost list removed entries should be moved to
312 * NULL if the entry should be freed.
313 * @param fReuseBuffer Flag whether a buffer should be reused if it has the same size
314 * @param ppbBuf Where to store the address of the buffer if an entry with the
315 * same size was found and fReuseBuffer is true.
316 *
317 * @note This function may return fewer bytes than requested because entries
318 * may be marked as non evictable if they are used for I/O at the
319 * moment.
320 */
321static size_t pdmBlkCacheEvictPagesFrom(PPDMBLKCACHEGLOBAL pCache, size_t cbData,
322 PPDMBLKLRULIST pListSrc, PPDMBLKLRULIST pGhostListDst,
323 bool fReuseBuffer, uint8_t **ppbBuffer)
324{
325 size_t cbEvicted = 0;
326
327 PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
328
329 AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
330 AssertMsg( !pGhostListDst
331 || (pGhostListDst == &pCache->LruRecentlyUsedOut),
332 ("Destination list must be NULL or the recently used but paged out list\n"));
333
334 if (fReuseBuffer)
335 {
336 AssertPtr(ppbBuffer);
337 *ppbBuffer = NULL;
338 }
339
340 /* Start deleting from the tail. */
341 PPDMBLKCACHEENTRY pEntry = pListSrc->pTail;
342
343 while ((cbEvicted < cbData) && pEntry)
344 {
345 PPDMBLKCACHEENTRY pCurr = pEntry;
346
347 pEntry = pEntry->pPrev;
348
349 /* We can't evict pages which are currently in progress or dirty but not in progress */
350 if ( !(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
351 && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
352 {
353 /* Ok eviction candidate. Grab the endpoint semaphore and check again
354 * because somebody else might have raced us. */
355 PPDMBLKCACHE pBlkCache = pCurr->pBlkCache;
356 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
357
358 if (!(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
359 && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
360 {
361 LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
362
363 if (fReuseBuffer && (pCurr->cbData == cbData))
364 {
365 STAM_COUNTER_INC(&pCache->StatBuffersReused);
366 *ppbBuffer = pCurr->pbData;
367 }
368 else if (pCurr->pbData)
369 RTMemPageFree(pCurr->pbData, pCurr->cbData);
370
371 pCurr->pbData = NULL;
372 cbEvicted += pCurr->cbData;
373
374 pdmBlkCacheEntryRemoveFromList(pCurr);
375 pdmBlkCacheSub(pCache, pCurr->cbData);
376
377 if (pGhostListDst)
378 {
379 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
380
381 PPDMBLKCACHEENTRY pGhostEntFree = pGhostListDst->pTail;
382
383 /* We have to remove the last entries from the paged out list. */
384 while ( ((pGhostListDst->cbCached + pCurr->cbData) > pCache->cbRecentlyUsedOutMax)
385 && pGhostEntFree)
386 {
387 PPDMBLKCACHEENTRY pFree = pGhostEntFree;
388 PPDMBLKCACHE pBlkCacheFree = pFree->pBlkCache;
389
390 pGhostEntFree = pGhostEntFree->pPrev;
391
392 RTSemRWRequestWrite(pBlkCacheFree->SemRWEntries, RT_INDEFINITE_WAIT);
393
394 if (ASMAtomicReadU32(&pFree->cRefs) == 0)
395 {
396 pdmBlkCacheEntryRemoveFromList(pFree);
397
398 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
399 RTAvlrU64Remove(pBlkCacheFree->pTree, pFree->Core.Key);
400 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
401
402 RTMemFree(pFree);
403 }
404
405 RTSemRWReleaseWrite(pBlkCacheFree->SemRWEntries);
406 }
407
408 if (pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax)
409 {
410 /* Couldn't remove enough entries. Delete */
411 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
412 RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
413 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
414
415 RTMemFree(pCurr);
416 }
417 else
418 pdmBlkCacheEntryAddToList(pGhostListDst, pCurr);
419 }
420 else
421 {
422 /* Delete the entry from the AVL tree it is assigned to. */
423 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
424 RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
425 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
426
427 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
428 RTMemFree(pCurr);
429 }
430 }
431
432 }
433 else
434 LogFlow(("Entry %#p (%u bytes) is still in progress and can't be evicted\n", pCurr, pCurr->cbData));
435 }
436
437 return cbEvicted;
438}
439
440static bool pdmBlkCacheReclaim(PPDMBLKCACHEGLOBAL pCache, size_t cbData, bool fReuseBuffer, uint8_t **ppbBuffer)
441{
442 size_t cbRemoved = 0;
443
444 if ((pCache->cbCached + cbData) < pCache->cbMax)
445 return true;
446 else if ((pCache->LruRecentlyUsedIn.cbCached + cbData) > pCache->cbRecentlyUsedInMax)
447 {
448 /* Try to evict as many bytes as possible from A1in */
449 cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruRecentlyUsedIn,
450 &pCache->LruRecentlyUsedOut, fReuseBuffer, ppbBuffer);
451
452 /*
453 * If it was not possible to remove enough entries
454 * try the frequently accessed cache.
455 */
456 if (cbRemoved < cbData)
457 {
458 Assert(!fReuseBuffer || !*ppbBuffer); /* It is not possible that we got a buffer with the correct size but we didn't freed enough data. */
459
460 /*
461 * If we removed something we can't pass the reuse buffer flag anymore because
462 * we don't need to evict that much data
463 */
464 if (!cbRemoved)
465 cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
466 NULL, fReuseBuffer, ppbBuffer);
467 else
468 cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData - cbRemoved, &pCache->LruFrequentlyUsed,
469 NULL, false, NULL);
470 }
471 }
472 else
473 {
474 /* We have to remove entries from frequently access list. */
475 cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
476 NULL, fReuseBuffer, ppbBuffer);
477 }
478
479 LogFlowFunc((": removed %u bytes, requested %u\n", cbRemoved, cbData));
480 return (cbRemoved >= cbData);
481}
482
483DECLINLINE(int) pdmBlkCacheEnqueue(PPDMBLKCACHE pBlkCache, uint64_t off, size_t cbXfer, PPDMBLKCACHEIOXFER pIoXfer)
484{
485 int rc = VINF_SUCCESS;
486
487 LogFlowFunc(("%s: Enqueuing hIoXfer=%#p enmXferDir=%d\n",
488 __FUNCTION__, pIoXfer, pIoXfer->enmXferDir));
489
490 switch (pBlkCache->enmType)
491 {
492 case PDMBLKCACHETYPE_DEV:
493 {
494 rc = pBlkCache->u.Dev.pfnXferEnqueue(pBlkCache->u.Dev.pDevIns,
495 pIoXfer->enmXferDir,
496 off, cbXfer,
497 &pIoXfer->SgBuf, pIoXfer);
498 break;
499 }
500 case PDMBLKCACHETYPE_DRV:
501 {
502 rc = pBlkCache->u.Drv.pfnXferEnqueue(pBlkCache->u.Drv.pDrvIns,
503 pIoXfer->enmXferDir,
504 off, cbXfer,
505 &pIoXfer->SgBuf, pIoXfer);
506 break;
507 }
508 case PDMBLKCACHETYPE_USB:
509 {
510 rc = pBlkCache->u.Usb.pfnXferEnqueue(pBlkCache->u.Usb.pUsbIns,
511 pIoXfer->enmXferDir,
512 off, cbXfer,
513 &pIoXfer->SgBuf, pIoXfer);
514 break;
515 }
516 case PDMBLKCACHETYPE_INTERNAL:
517 {
518 rc = pBlkCache->u.Int.pfnXferEnqueue(pBlkCache->u.Int.pvUser,
519 pIoXfer->enmXferDir,
520 off, cbXfer,
521 &pIoXfer->SgBuf, pIoXfer);
522 break;
523 }
524 default:
525 AssertMsgFailed(("Unknown block cache type!\n"));
526 }
527
528 LogFlowFunc(("%s: returns rc=%Rrc\n", __FUNCTION__, rc));
529 return rc;
530}
531
532/**
533 * Initiates a read I/O task for the given entry.
534 *
535 * @returns VBox status code.
536 * @param pEntry The entry to fetch the data to.
537 */
538static int pdmBlkCacheEntryReadFromMedium(PPDMBLKCACHEENTRY pEntry)
539{
540 PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
541 LogFlowFunc((": Reading data into cache entry %#p\n", pEntry));
542
543 /* Make sure no one evicts the entry while it is accessed. */
544 pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
545
546 PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
547 if (RT_UNLIKELY(!pIoXfer))
548 return VERR_NO_MEMORY;
549
550 AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
551
552 pIoXfer->fIoCache = true;
553 pIoXfer->pEntry = pEntry;
554 pIoXfer->SgSeg.pvSeg = pEntry->pbData;
555 pIoXfer->SgSeg.cbSeg = pEntry->cbData;
556 pIoXfer->enmXferDir = PDMBLKCACHEXFERDIR_READ;
557 RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
558
559 return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pEntry->cbData, pIoXfer);
560}
561
562/**
563 * Initiates a write I/O task for the given entry.
564 *
565 * @returns nothing.
566 * @param pEntry The entry to read the data from.
567 */
568static int pdmBlkCacheEntryWriteToMedium(PPDMBLKCACHEENTRY pEntry)
569{
570 PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
571 LogFlowFunc((": Writing data from cache entry %#p\n", pEntry));
572
573 /* Make sure no one evicts the entry while it is accessed. */
574 pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
575
576 PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
577 if (RT_UNLIKELY(!pIoXfer))
578 return VERR_NO_MEMORY;
579
580 AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
581
582 pIoXfer->fIoCache = true;
583 pIoXfer->pEntry = pEntry;
584 pIoXfer->SgSeg.pvSeg = pEntry->pbData;
585 pIoXfer->SgSeg.cbSeg = pEntry->cbData;
586 pIoXfer->enmXferDir = PDMBLKCACHEXFERDIR_WRITE;
587 RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
588
589 return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pEntry->cbData, pIoXfer);
590}
591
592/**
593 * Passthrough a part of a request directly to the I/O manager
594 * handling the endpoint.
595 *
596 * @returns VBox status code.
597 * @param pEndpoint The endpoint.
598 * @param pTask The task.
599 * @param pIoMemCtx The I/O memory context to use.
600 * @param offStart Offset to start transfer from.
601 * @param cbData Amount of data to transfer.
602 * @param enmTransferType The transfer type (read/write)
603 */
604static int pdmBlkCacheRequestPassthrough(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
605 PRTSGBUF pSgBuf, uint64_t offStart, size_t cbData,
606 PDMBLKCACHEXFERDIR enmXferDir)
607{
608
609 PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
610 if (RT_UNLIKELY(!pIoXfer))
611 return VERR_NO_MEMORY;
612
613 ASMAtomicIncU32(&pReq->cXfersPending);
614 pIoXfer->fIoCache = false;
615 pIoXfer->pReq = pReq;
616 pIoXfer->enmXferDir = enmXferDir;
617 if (pSgBuf)
618 {
619 RTSgBufClone(&pIoXfer->SgBuf, pSgBuf);
620 RTSgBufAdvance(pSgBuf, cbData);
621 }
622
623 return pdmBlkCacheEnqueue(pBlkCache, offStart, cbData, pIoXfer);
624}
625
626/**
627 * Commit a single dirty entry to the endpoint
628 *
629 * @returns nothing
630 * @param pEntry The entry to commit.
631 */
632static void pdmBlkCacheEntryCommit(PPDMBLKCACHEENTRY pEntry)
633{
634 AssertMsg( (pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY)
635 && !(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
636 ("Invalid flags set for entry %#p\n", pEntry));
637
638 pdmBlkCacheEntryWriteToMedium(pEntry);
639}
640
641/**
642 * Commit all dirty entries for a single endpoint.
643 *
644 * @returns nothing.
645 * @param pBlkCache The endpoint cache to commit.
646 */
647static void pdmBlkCacheCommit(PPDMBLKCACHE pBlkCache)
648{
649 uint32_t cbCommitted = 0;
650
651 /* Return if the cache was suspended. */
652 if (pBlkCache->fSuspended)
653 return;
654
655 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
656
657 /* The list is moved to a new header to reduce locking overhead. */
658 RTLISTNODE ListDirtyNotCommitted;
659 RTSPINLOCKTMP Tmp;
660
661 RTListInit(&ListDirtyNotCommitted);
662 RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
663 RTListMove(&ListDirtyNotCommitted, &pBlkCache->ListDirtyNotCommitted);
664 RTSpinlockRelease(pBlkCache->LockList, &Tmp);
665
666 if (!RTListIsEmpty(&ListDirtyNotCommitted))
667 {
668 PPDMBLKCACHEENTRY pEntry = RTListGetFirst(&ListDirtyNotCommitted, PDMBLKCACHEENTRY, NodeNotCommitted);
669
670 while (!RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted))
671 {
672 PPDMBLKCACHEENTRY pNext = RTListNodeGetNext(&pEntry->NodeNotCommitted, PDMBLKCACHEENTRY,
673 NodeNotCommitted);
674 pdmBlkCacheEntryCommit(pEntry);
675 cbCommitted += pEntry->cbData;
676 RTListNodeRemove(&pEntry->NodeNotCommitted);
677 pEntry = pNext;
678 }
679
680 /* Commit the last endpoint */
681 Assert(RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted));
682 pdmBlkCacheEntryCommit(pEntry);
683 RTListNodeRemove(&pEntry->NodeNotCommitted);
684 AssertMsg(RTListIsEmpty(&ListDirtyNotCommitted),
685 ("Committed all entries but list is not empty\n"));
686 }
687
688 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
689 AssertMsg(pBlkCache->pCache->cbDirty >= cbCommitted,
690 ("Number of committed bytes exceeds number of dirty bytes\n"));
691 uint32_t cbDirtyOld = ASMAtomicSubU32(&pBlkCache->pCache->cbDirty, cbCommitted);
692
693 /* Reset the commit timer if we don't have any dirty bits. */
694 if ( !(cbDirtyOld - cbCommitted)
695 && pBlkCache->pCache->u32CommitTimeoutMs != 0)
696 TMTimerStop(pBlkCache->pCache->pTimerCommit);
697}
698
699/**
700 * Commit all dirty entries in the cache.
701 *
702 * @returns nothing.
703 * @param pCache The global cache instance.
704 */
705static void pdmBlkCacheCommitDirtyEntries(PPDMBLKCACHEGLOBAL pCache)
706{
707 bool fCommitInProgress = ASMAtomicXchgBool(&pCache->fCommitInProgress, true);
708
709 if (!fCommitInProgress)
710 {
711 pdmBlkCacheLockEnter(pCache);
712 Assert(!RTListIsEmpty(&pCache->ListUsers));
713
714 PPDMBLKCACHE pBlkCache = RTListGetFirst(&pCache->ListUsers, PDMBLKCACHE, NodeCacheUser);
715 AssertPtr(pBlkCache);
716
717 while (!RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser))
718 {
719 pdmBlkCacheCommit(pBlkCache);
720
721 pBlkCache = RTListNodeGetNext(&pBlkCache->NodeCacheUser, PDMBLKCACHE,
722 NodeCacheUser);
723 }
724
725 /* Commit the last endpoint */
726 Assert(RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser));
727 pdmBlkCacheCommit(pBlkCache);
728
729 pdmBlkCacheLockLeave(pCache);
730 ASMAtomicWriteBool(&pCache->fCommitInProgress, false);
731 }
732}
733
734/**
735 * Adds the given entry as a dirty to the cache.
736 *
737 * @returns Flag whether the amount of dirty bytes in the cache exceeds the threshold
738 * @param pBlkCache The endpoint cache the entry belongs to.
739 * @param pEntry The entry to add.
740 */
741static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
742{
743 bool fDirtyBytesExceeded = false;
744 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
745
746 /* If the commit timer is disabled we commit right away. */
747 if (pCache->u32CommitTimeoutMs == 0)
748 {
749 pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
750 pdmBlkCacheEntryCommit(pEntry);
751 }
752 else if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY))
753 {
754 pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
755
756 RTSPINLOCKTMP Tmp;
757 RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
758 RTListAppend(&pBlkCache->ListDirtyNotCommitted, &pEntry->NodeNotCommitted);
759 RTSpinlockRelease(pBlkCache->LockList, &Tmp);
760
761 uint32_t cbDirty = ASMAtomicAddU32(&pCache->cbDirty, pEntry->cbData);
762
763 /* Prevent committing if the VM was suspended. */
764 if (RT_LIKELY(!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended)))
765 fDirtyBytesExceeded = (cbDirty + pEntry->cbData >= pCache->cbCommitDirtyThreshold);
766 else if (!cbDirty && pCache->u32CommitTimeoutMs > 0)
767 {
768 /* Arm the commit timer. */
769 TMTimerSetMillies(pCache->pTimerCommit, pCache->u32CommitTimeoutMs);
770 }
771 }
772
773 return fDirtyBytesExceeded;
774}
775
776static PPDMBLKCACHE pdmR3BlkCacheFindById(PPDMBLKCACHEGLOBAL pBlkCacheGlobal, const char *pcszId)
777{
778 bool fFound = false;
779 PPDMBLKCACHE pBlkCache = NULL;
780
781 RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
782 {
783 if (!RTStrCmp(pBlkCache->pszId, pcszId))
784 {
785 fFound = true;
786 break;
787 }
788 }
789
790 return fFound ? pBlkCache : NULL;
791}
792
793/**
794 * Commit timer callback.
795 */
796static void pdmBlkCacheCommitTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
797{
798 PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
799
800 LogFlowFunc(("Commit interval expired, commiting dirty entries\n"));
801
802 if ( ASMAtomicReadU32(&pCache->cbDirty) > 0
803 && !ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
804 pdmBlkCacheCommitDirtyEntries(pCache);
805
806 LogFlowFunc(("Entries committed, going to sleep\n"));
807}
808
809static DECLCALLBACK(int) pdmR3BlkCacheSaveExec(PVM pVM, PSSMHANDLE pSSM)
810{
811 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
812
813 AssertPtr(pBlkCacheGlobal);
814
815 pdmBlkCacheLockEnter(pBlkCacheGlobal);
816
817 SSMR3PutU32(pSSM, pBlkCacheGlobal->cRefs);
818
819 /* Go through the list and save all dirty entries. */
820 PPDMBLKCACHE pBlkCache;
821 RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
822 {
823 uint32_t cEntries = 0;
824 PPDMBLKCACHEENTRY pEntry;
825
826 RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
827 SSMR3PutU32(pSSM, strlen(pBlkCache->pszId));
828 SSMR3PutStrZ(pSSM, pBlkCache->pszId);
829
830 /* Count the number of entries to safe. */
831 RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
832 {
833 cEntries++;
834 }
835
836 SSMR3PutU32(pSSM, cEntries);
837
838 /* Walk the list of all dirty entries and save them. */
839 RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
840 {
841 /* A few sanity checks. */
842 AssertMsg(!pEntry->cRefs, ("The entry is still referenced\n"));
843 AssertMsg(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY, ("Entry is not dirty\n"));
844 AssertMsg(!(pEntry->fFlags & ~PDMBLKCACHE_ENTRY_IS_DIRTY), ("Invalid flags set\n"));
845 AssertMsg(!pEntry->pWaitingHead && !pEntry->pWaitingTail, ("There are waiting requests\n"));
846 AssertMsg( pEntry->pList == &pBlkCacheGlobal->LruRecentlyUsedIn
847 || pEntry->pList == &pBlkCacheGlobal->LruFrequentlyUsed,
848 ("Invalid list\n"));
849 AssertMsg(pEntry->cbData == pEntry->Core.KeyLast - pEntry->Core.Key + 1,
850 ("Size and range do not match\n"));
851
852 /* Save */
853 SSMR3PutU64(pSSM, pEntry->Core.Key);
854 SSMR3PutU32(pSSM, pEntry->cbData);
855 SSMR3PutMem(pSSM, pEntry->pbData, pEntry->cbData);
856 }
857
858 RTSemRWReleaseRead(pBlkCache->SemRWEntries);
859 }
860
861 pdmBlkCacheLockLeave(pBlkCacheGlobal);
862
863 /* Terminator */
864 return SSMR3PutU32(pSSM, UINT32_MAX);
865}
866
867static DECLCALLBACK(int) pdmR3BlkCacheLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
868{
869 int rc = VINF_SUCCESS;
870 uint32_t cRefs;
871 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
872
873 AssertPtr(pBlkCacheGlobal);
874
875 pdmBlkCacheLockEnter(pBlkCacheGlobal);
876
877 if (uVersion != PDM_BLK_CACHE_SAVED_STATE_VERSION)
878 return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
879
880 SSMR3GetU32(pSSM, &cRefs);
881
882 if (cRefs == pBlkCacheGlobal->cRefs)
883 {
884 char *pszId = NULL;
885
886 while ( cRefs > 0
887 && RT_SUCCESS(rc))
888 {
889 PPDMBLKCACHE pBlkCache = NULL;
890 uint32_t cbId = 0;
891
892 SSMR3GetU32(pSSM, &cbId);
893 Assert(cbId > 0);
894
895 cbId++; /* Include terminator */
896 pszId = (char *)RTMemAllocZ(cbId * sizeof(char));
897 if (!pszId)
898 {
899 rc = VERR_NO_MEMORY;
900 break;
901 }
902
903 rc = SSMR3GetStrZ(pSSM, pszId, cbId);
904 AssertRC(rc);
905
906 /* Search for the block cache with the provided id. */
907 pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pszId);
908 if (!pBlkCache)
909 {
910 rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
911 N_("The VM is missing a block device. Please make sure the source and target VMs have compatible storage configurations"));
912 break;
913 }
914
915 RTStrFree(pszId);
916 pszId = NULL;
917
918 /* Get the entries */
919 uint32_t cEntries;
920 SSMR3GetU32(pSSM, &cEntries);
921
922 while (cEntries > 0)
923 {
924 PPDMBLKCACHEENTRY pEntry;
925 uint64_t off;
926 uint32_t cbEntry;
927
928 SSMR3GetU64(pSSM, &off);
929 SSMR3GetU32(pSSM, &cbEntry);
930
931 pEntry = pdmBlkCacheEntryAlloc(pBlkCache, off, cbEntry, NULL);
932 if (!pEntry)
933 {
934 rc = VERR_NO_MEMORY;
935 break;
936 }
937
938 rc = SSMR3GetMem(pSSM, pEntry->pbData, cbEntry);
939 if (RT_FAILURE(rc))
940 {
941 RTMemFree(pEntry->pbData);
942 RTMemFree(pEntry);
943 break;
944 }
945
946 /* Insert into the tree. */
947 bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
948 Assert(fInserted);
949
950 /* Add to the dirty list. */
951 pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
952 pdmBlkCacheEntryAddToList(&pBlkCacheGlobal->LruRecentlyUsedIn, pEntry);
953 pdmBlkCacheAdd(pBlkCacheGlobal, cbEntry);
954 pdmBlkCacheEntryRelease(pEntry);
955 cEntries--;
956 }
957
958 cRefs--;
959 }
960
961 if (pszId)
962 RTStrFree(pszId);
963 }
964 else
965 rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
966 N_("The VM is missing a block device. Please make sure the source and target VMs have compatible storage configurations"));
967
968 pdmBlkCacheLockLeave(pBlkCacheGlobal);
969
970 if (RT_SUCCESS(rc))
971 {
972 uint32_t u32 = 0;
973 rc = SSMR3GetU32(pSSM, &u32);
974 if (RT_SUCCESS(rc))
975 AssertMsgReturn(u32 == UINT32_MAX, ("%#x\n", u32), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
976 }
977
978 return rc;
979}
980
981int pdmR3BlkCacheInit(PVM pVM)
982{
983 int rc = VINF_SUCCESS;
984 PUVM pUVM = pVM->pUVM;
985 PPDMBLKCACHEGLOBAL pBlkCacheGlobal;
986
987 LogFlowFunc((": pVM=%p\n", pVM));
988
989 VM_ASSERT_EMT(pVM);
990
991 PCFGMNODE pCfgRoot = CFGMR3GetRoot(pVM);
992 PCFGMNODE pCfgBlkCache = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "BlkCache");
993
994 pBlkCacheGlobal = (PPDMBLKCACHEGLOBAL)RTMemAllocZ(sizeof(PDMBLKCACHEGLOBAL));
995 if (!pBlkCacheGlobal)
996 return VERR_NO_MEMORY;
997
998 RTListInit(&pBlkCacheGlobal->ListUsers);
999 pBlkCacheGlobal->pVM = pVM;
1000 pBlkCacheGlobal->cRefs = 0;
1001 pBlkCacheGlobal->cbCached = 0;
1002 pBlkCacheGlobal->fCommitInProgress = false;
1003
1004 /* Initialize members */
1005 pBlkCacheGlobal->LruRecentlyUsedIn.pHead = NULL;
1006 pBlkCacheGlobal->LruRecentlyUsedIn.pTail = NULL;
1007 pBlkCacheGlobal->LruRecentlyUsedIn.cbCached = 0;
1008
1009 pBlkCacheGlobal->LruRecentlyUsedOut.pHead = NULL;
1010 pBlkCacheGlobal->LruRecentlyUsedOut.pTail = NULL;
1011 pBlkCacheGlobal->LruRecentlyUsedOut.cbCached = 0;
1012
1013 pBlkCacheGlobal->LruFrequentlyUsed.pHead = NULL;
1014 pBlkCacheGlobal->LruFrequentlyUsed.pTail = NULL;
1015 pBlkCacheGlobal->LruFrequentlyUsed.cbCached = 0;
1016
1017 do
1018 {
1019 rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheSize", &pBlkCacheGlobal->cbMax, 5 * _1M);
1020 AssertLogRelRCBreak(rc);
1021 LogFlowFunc(("Maximum number of bytes cached %u\n", pBlkCacheGlobal->cbMax));
1022
1023 pBlkCacheGlobal->cbRecentlyUsedInMax = (pBlkCacheGlobal->cbMax / 100) * 25; /* 25% of the buffer size */
1024 pBlkCacheGlobal->cbRecentlyUsedOutMax = (pBlkCacheGlobal->cbMax / 100) * 50; /* 50% of the buffer size */
1025 LogFlowFunc(("cbRecentlyUsedInMax=%u cbRecentlyUsedOutMax=%u\n",
1026 pBlkCacheGlobal->cbRecentlyUsedInMax, pBlkCacheGlobal->cbRecentlyUsedOutMax));
1027
1028 /** @todo r=aeichner: Experiment to find optimal default values */
1029 rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitIntervalMs", &pBlkCacheGlobal->u32CommitTimeoutMs, 10000 /* 10sec */);
1030 AssertLogRelRCBreak(rc);
1031 rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitThreshold", &pBlkCacheGlobal->cbCommitDirtyThreshold, pBlkCacheGlobal->cbMax / 2);
1032 AssertLogRelRCBreak(rc);
1033 } while (0);
1034
1035 if (RT_SUCCESS(rc))
1036 {
1037 STAMR3Register(pVM, &pBlkCacheGlobal->cbMax,
1038 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1039 "/PDM/BlkCache/cbMax",
1040 STAMUNIT_BYTES,
1041 "Maximum cache size");
1042 STAMR3Register(pVM, &pBlkCacheGlobal->cbCached,
1043 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1044 "/PDM/BlkCache/cbCached",
1045 STAMUNIT_BYTES,
1046 "Currently used cache");
1047 STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedIn.cbCached,
1048 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1049 "/PDM/BlkCache/cbCachedMruIn",
1050 STAMUNIT_BYTES,
1051 "Number of bytes cached in MRU list");
1052 STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedOut.cbCached,
1053 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1054 "/PDM/BlkCache/cbCachedMruOut",
1055 STAMUNIT_BYTES,
1056 "Number of bytes cached in FRU list");
1057 STAMR3Register(pVM, &pBlkCacheGlobal->LruFrequentlyUsed.cbCached,
1058 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1059 "/PDM/BlkCache/cbCachedFru",
1060 STAMUNIT_BYTES,
1061 "Number of bytes cached in FRU ghost list");
1062
1063#ifdef VBOX_WITH_STATISTICS
1064 STAMR3Register(pVM, &pBlkCacheGlobal->cHits,
1065 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1066 "/PDM/BlkCache/CacheHits",
1067 STAMUNIT_COUNT, "Number of hits in the cache");
1068 STAMR3Register(pVM, &pBlkCacheGlobal->cPartialHits,
1069 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1070 "/PDM/BlkCache/CachePartialHits",
1071 STAMUNIT_COUNT, "Number of partial hits in the cache");
1072 STAMR3Register(pVM, &pBlkCacheGlobal->cMisses,
1073 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1074 "/PDM/BlkCache/CacheMisses",
1075 STAMUNIT_COUNT, "Number of misses when accessing the cache");
1076 STAMR3Register(pVM, &pBlkCacheGlobal->StatRead,
1077 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1078 "/PDM/BlkCache/CacheRead",
1079 STAMUNIT_BYTES, "Number of bytes read from the cache");
1080 STAMR3Register(pVM, &pBlkCacheGlobal->StatWritten,
1081 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1082 "/PDM/BlkCache/CacheWritten",
1083 STAMUNIT_BYTES, "Number of bytes written to the cache");
1084 STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeGet,
1085 STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
1086 "/PDM/BlkCache/CacheTreeGet",
1087 STAMUNIT_TICKS_PER_CALL, "Time taken to access an entry in the tree");
1088 STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeInsert,
1089 STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
1090 "/PDM/BlkCache/CacheTreeInsert",
1091 STAMUNIT_TICKS_PER_CALL, "Time taken to insert an entry in the tree");
1092 STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeRemove,
1093 STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
1094 "/PDM/BlkCache/CacheTreeRemove",
1095 STAMUNIT_TICKS_PER_CALL, "Time taken to remove an entry an the tree");
1096 STAMR3Register(pVM, &pBlkCacheGlobal->StatBuffersReused,
1097 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1098 "/PDM/BlkCache/CacheBuffersReused",
1099 STAMUNIT_COUNT, "Number of times a buffer could be reused");
1100#endif
1101
1102 /* Initialize the critical section */
1103 rc = RTCritSectInit(&pBlkCacheGlobal->CritSect);
1104 }
1105
1106 if (RT_SUCCESS(rc))
1107 {
1108 /* Create the commit timer */
1109 if (pBlkCacheGlobal->u32CommitTimeoutMs > 0)
1110 rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL,
1111 pdmBlkCacheCommitTimerCallback,
1112 pBlkCacheGlobal,
1113 "BlkCache-Commit",
1114 &pBlkCacheGlobal->pTimerCommit);
1115
1116 if (RT_SUCCESS(rc))
1117 {
1118 /* Register saved state handler. */
1119 rc = SSMR3RegisterInternal(pVM, "pdmblkcache", 0, PDM_BLK_CACHE_SAVED_STATE_VERSION, pBlkCacheGlobal->cbMax,
1120 NULL, NULL, NULL,
1121 NULL, pdmR3BlkCacheSaveExec, NULL,
1122 NULL, pdmR3BlkCacheLoadExec, NULL);
1123 if (RT_SUCCESS(rc))
1124 {
1125 LogRel(("BlkCache: Cache successfully initialised. Cache size is %u bytes\n", pBlkCacheGlobal->cbMax));
1126 LogRel(("BlkCache: Cache commit interval is %u ms\n", pBlkCacheGlobal->u32CommitTimeoutMs));
1127 LogRel(("BlkCache: Cache commit threshold is %u bytes\n", pBlkCacheGlobal->cbCommitDirtyThreshold));
1128 pUVM->pdm.s.pBlkCacheGlobal = pBlkCacheGlobal;
1129 return VINF_SUCCESS;
1130 }
1131 }
1132
1133 RTCritSectDelete(&pBlkCacheGlobal->CritSect);
1134 }
1135
1136 if (pBlkCacheGlobal)
1137 RTMemFree(pBlkCacheGlobal);
1138
1139 LogFlowFunc((": returns rc=%Rrc\n", pVM, rc));
1140 return rc;
1141}
1142
1143void pdmR3BlkCacheTerm(PVM pVM)
1144{
1145 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1146
1147 if (pBlkCacheGlobal)
1148 {
1149 /* Make sure no one else uses the cache now */
1150 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1151
1152 /* Cleanup deleting all cache entries waiting for in progress entries to finish. */
1153 pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedIn);
1154 pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedOut);
1155 pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruFrequentlyUsed);
1156
1157 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1158
1159 RTCritSectDelete(&pBlkCacheGlobal->CritSect);
1160 RTMemFree(pBlkCacheGlobal);
1161 pVM->pUVM->pdm.s.pBlkCacheGlobal = NULL;
1162 }
1163}
1164
1165int pdmR3BlkCacheResume(PVM pVM)
1166{
1167 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1168
1169 LogFlowFunc(("pVM=%#p\n", pVM));
1170
1171 if ( pBlkCacheGlobal
1172 && ASMAtomicXchgBool(&pBlkCacheGlobal->fIoErrorVmSuspended, false))
1173 {
1174 /* The VM was suspended because of an I/O error, commit all dirty entries. */
1175 pdmBlkCacheCommitDirtyEntries(pBlkCacheGlobal);
1176 }
1177
1178 return VINF_SUCCESS;
1179}
1180
1181static int pdmR3BlkCacheRetain(PVM pVM, PPPDMBLKCACHE ppBlkCache, const char *pcszId)
1182{
1183 int rc = VINF_SUCCESS;
1184 PPDMBLKCACHE pBlkCache = NULL;
1185 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1186
1187 if (!pBlkCacheGlobal)
1188 return VERR_NOT_SUPPORTED;
1189
1190 /*
1191 * Check that no other user cache has the same id first,
1192 * Unique id's are necessary in case the state is saved.
1193 */
1194 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1195
1196 pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pcszId);
1197
1198 if (!pBlkCache)
1199 {
1200 pBlkCache = (PPDMBLKCACHE)RTMemAllocZ(sizeof(PDMBLKCACHE));
1201
1202 if (pBlkCache)
1203 pBlkCache->pszId = RTStrDup(pcszId);
1204
1205 if ( pBlkCache
1206 && pBlkCache->pszId)
1207 {
1208 pBlkCache->fSuspended = false;
1209 pBlkCache->pCache = pBlkCacheGlobal;
1210 RTListInit(&pBlkCache->ListDirtyNotCommitted);
1211
1212 rc = RTSpinlockCreate(&pBlkCache->LockList);
1213 if (RT_SUCCESS(rc))
1214 {
1215 rc = RTSemRWCreate(&pBlkCache->SemRWEntries);
1216 if (RT_SUCCESS(rc))
1217 {
1218 pBlkCache->pTree = (PAVLRU64TREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
1219 if (pBlkCache->pTree)
1220 {
1221#ifdef VBOX_WITH_STATISTICS
1222 STAMR3RegisterF(pBlkCacheGlobal->pVM, &pBlkCache->StatWriteDeferred,
1223 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1224 STAMUNIT_COUNT, "Number of deferred writes",
1225 "/PDM/BlkCache/%s/Cache/DeferredWrites", pBlkCache->pszId);
1226#endif
1227
1228 /* Add to the list of users. */
1229 pBlkCacheGlobal->cRefs++;
1230 RTListAppend(&pBlkCacheGlobal->ListUsers, &pBlkCache->NodeCacheUser);
1231 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1232
1233 *ppBlkCache = pBlkCache;
1234 LogFlowFunc(("returns success\n"));
1235 return VINF_SUCCESS;
1236 }
1237 else
1238 rc = VERR_NO_MEMORY;
1239
1240 RTSemRWDestroy(pBlkCache->SemRWEntries);
1241 }
1242
1243 RTSpinlockDestroy(pBlkCache->LockList);
1244 }
1245
1246 RTStrFree(pBlkCache->pszId);
1247 }
1248 else
1249 rc = VERR_NO_MEMORY;
1250
1251 if (pBlkCache)
1252 RTMemFree(pBlkCache);
1253 }
1254 else
1255 rc = VERR_ALREADY_EXISTS;
1256
1257 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1258
1259 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1260 return rc;
1261}
1262
1263VMMR3DECL(int) PDMR3BlkCacheRetainDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
1264 PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
1265 PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
1266 const char *pcszId)
1267{
1268 int rc = VINF_SUCCESS;
1269 PPDMBLKCACHE pBlkCache;
1270
1271 rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1272 if (RT_SUCCESS(rc))
1273 {
1274 pBlkCache->enmType = PDMBLKCACHETYPE_DRV;
1275 pBlkCache->u.Drv.pfnXferComplete = pfnXferComplete;
1276 pBlkCache->u.Drv.pfnXferEnqueue = pfnXferEnqueue;
1277 pBlkCache->u.Drv.pDrvIns = pDrvIns;
1278 *ppBlkCache = pBlkCache;
1279 }
1280
1281 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1282 return rc;
1283}
1284
1285VMMR3DECL(int) PDMR3BlkCacheRetainDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMBLKCACHE ppBlkCache,
1286 PFNPDMBLKCACHEXFERCOMPLETEDEV pfnXferComplete,
1287 PFNPDMBLKCACHEXFERENQUEUEDEV pfnXferEnqueue,
1288 const char *pcszId)
1289{
1290 int rc = VINF_SUCCESS;
1291 PPDMBLKCACHE pBlkCache;
1292
1293 rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1294 if (RT_SUCCESS(rc))
1295 {
1296 pBlkCache->enmType = PDMBLKCACHETYPE_DEV;
1297 pBlkCache->u.Dev.pfnXferComplete = pfnXferComplete;
1298 pBlkCache->u.Dev.pfnXferEnqueue = pfnXferEnqueue;
1299 pBlkCache->u.Dev.pDevIns = pDevIns;
1300 *ppBlkCache = pBlkCache;
1301 }
1302
1303 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1304 return rc;
1305
1306}
1307
1308VMMR3DECL(int) PDMR3BlkCacheRetainUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMBLKCACHE ppBlkCache,
1309 PFNPDMBLKCACHEXFERCOMPLETEUSB pfnXferComplete,
1310 PFNPDMBLKCACHEXFERENQUEUEUSB pfnXferEnqueue,
1311 const char *pcszId)
1312{
1313 int rc = VINF_SUCCESS;
1314 PPDMBLKCACHE pBlkCache;
1315
1316 rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1317 if (RT_SUCCESS(rc))
1318 {
1319 pBlkCache->enmType = PDMBLKCACHETYPE_USB;
1320 pBlkCache->u.Usb.pfnXferComplete = pfnXferComplete;
1321 pBlkCache->u.Usb.pfnXferEnqueue = pfnXferEnqueue;
1322 pBlkCache->u.Usb.pUsbIns = pUsbIns;
1323 *ppBlkCache = pBlkCache;
1324 }
1325
1326 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1327 return rc;
1328
1329}
1330
1331VMMR3DECL(int) PDMR3BlkCacheRetainInt(PVM pVM, void *pvUser, PPPDMBLKCACHE ppBlkCache,
1332 PFNPDMBLKCACHEXFERCOMPLETEINT pfnXferComplete,
1333 PFNPDMBLKCACHEXFERENQUEUEINT pfnXferEnqueue,
1334 const char *pcszId)
1335{
1336 int rc = VINF_SUCCESS;
1337 PPDMBLKCACHE pBlkCache;
1338
1339 rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1340 if (RT_SUCCESS(rc))
1341 {
1342 pBlkCache->enmType = PDMBLKCACHETYPE_INTERNAL;
1343 pBlkCache->u.Int.pfnXferComplete = pfnXferComplete;
1344 pBlkCache->u.Int.pfnXferEnqueue = pfnXferEnqueue;
1345 pBlkCache->u.Int.pvUser = pvUser;
1346 *ppBlkCache = pBlkCache;
1347 }
1348
1349 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1350 return rc;
1351
1352}
1353
1354/**
1355 * Callback for the AVL destroy routine. Frees a cache entry for this endpoint.
1356 *
1357 * @returns IPRT status code.
1358 * @param pNode The node to destroy.
1359 * @param pvUser Opaque user data.
1360 */
1361static int pdmBlkCacheEntryDestroy(PAVLRU64NODECORE pNode, void *pvUser)
1362{
1363 PPDMBLKCACHEENTRY pEntry = (PPDMBLKCACHEENTRY)pNode;
1364 PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
1365 PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
1366
1367 while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
1368 {
1369 /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
1370 pdmBlkCacheEntryRef(pEntry);
1371 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1372 pdmBlkCacheLockLeave(pCache);
1373
1374 RTThreadSleep(250);
1375
1376 /* Re-enter all locks */
1377 pdmBlkCacheLockEnter(pCache);
1378 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1379 pdmBlkCacheEntryRelease(pEntry);
1380 }
1381
1382 AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
1383 ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
1384
1385 bool fUpdateCache = pEntry->pList == &pCache->LruFrequentlyUsed
1386 || pEntry->pList == &pCache->LruRecentlyUsedIn;
1387
1388 pdmBlkCacheEntryRemoveFromList(pEntry);
1389
1390 if (fUpdateCache)
1391 pdmBlkCacheSub(pCache, pEntry->cbData);
1392
1393 RTMemPageFree(pEntry->pbData, pEntry->cbData);
1394 RTMemFree(pEntry);
1395
1396 return VINF_SUCCESS;
1397}
1398
1399/**
1400 * Destroys all cache resources used by the given endpoint.
1401 *
1402 * @returns nothing.
1403 * @param pEndpoint The endpoint to the destroy.
1404 */
1405VMMR3DECL(void) PDMR3BlkCacheRelease(PPDMBLKCACHE pBlkCache)
1406{
1407 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1408
1409 /*
1410 * Commit all dirty entries now (they are waited on for completion during the
1411 * destruction of the AVL tree below).
1412 * The exception is if the VM was paused because of an I/O error before.
1413 */
1414 if (!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
1415 pdmBlkCacheCommit(pBlkCache);
1416
1417 /* Make sure nobody is accessing the cache while we delete the tree. */
1418 pdmBlkCacheLockEnter(pCache);
1419 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1420 RTAvlrU64Destroy(pBlkCache->pTree, pdmBlkCacheEntryDestroy, pCache);
1421 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1422
1423 RTSpinlockDestroy(pBlkCache->LockList);
1424
1425 pCache->cRefs--;
1426 RTListNodeRemove(&pBlkCache->NodeCacheUser);
1427
1428 pdmBlkCacheLockLeave(pCache);
1429
1430 RTSemRWDestroy(pBlkCache->SemRWEntries);
1431
1432#ifdef VBOX_WITH_STATISTICS
1433 STAMR3Deregister(pCache->pVM, &pBlkCache->StatWriteDeferred);
1434#endif
1435
1436 RTStrFree(pBlkCache->pszId);
1437 RTMemFree(pBlkCache);
1438}
1439
1440VMMR3DECL(void) PDMR3BlkCacheReleaseDevice(PVM pVM, PPDMDEVINS pDevIns)
1441{
1442 LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDevIns));
1443
1444 /*
1445 * Validate input.
1446 */
1447 if (!pDevIns)
1448 return;
1449 VM_ASSERT_EMT(pVM);
1450
1451 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1452 PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1453
1454 /* Return silently if not supported. */
1455 if (!pBlkCacheGlobal)
1456 return;
1457
1458 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1459
1460 RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1461 {
1462 if ( pBlkCache->enmType == PDMBLKCACHETYPE_DEV
1463 && pBlkCache->u.Dev.pDevIns == pDevIns)
1464 PDMR3BlkCacheRelease(pBlkCache);
1465 }
1466
1467 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1468}
1469
1470VMMR3DECL(void) PDMR3BlkCacheReleaseDriver(PVM pVM, PPDMDRVINS pDrvIns)
1471{
1472 LogFlow(("%s: pDrvIns=%p\n", __FUNCTION__, pDrvIns));
1473
1474 /*
1475 * Validate input.
1476 */
1477 if (!pDrvIns)
1478 return;
1479 VM_ASSERT_EMT(pVM);
1480
1481 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1482 PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1483
1484 /* Return silently if not supported. */
1485 if (!pBlkCacheGlobal)
1486 return;
1487
1488 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1489
1490 RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1491 {
1492 if ( pBlkCache->enmType == PDMBLKCACHETYPE_DRV
1493 && pBlkCache->u.Drv.pDrvIns == pDrvIns)
1494 PDMR3BlkCacheRelease(pBlkCache);
1495 }
1496
1497 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1498}
1499
1500VMMR3DECL(void) PDMR3BlkCacheReleaseUsb(PVM pVM, PPDMUSBINS pUsbIns)
1501{
1502 LogFlow(("%s: pUsbIns=%p\n", __FUNCTION__, pUsbIns));
1503
1504 /*
1505 * Validate input.
1506 */
1507 if (!pUsbIns)
1508 return;
1509 VM_ASSERT_EMT(pVM);
1510
1511 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1512 PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1513
1514 /* Return silently if not supported. */
1515 if (!pBlkCacheGlobal)
1516 return;
1517
1518 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1519
1520 RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1521 {
1522 if ( pBlkCache->enmType == PDMBLKCACHETYPE_USB
1523 && pBlkCache->u.Usb.pUsbIns == pUsbIns)
1524 PDMR3BlkCacheRelease(pBlkCache);
1525 }
1526
1527 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1528}
1529
1530static PPDMBLKCACHEENTRY pdmBlkCacheGetCacheEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off)
1531{
1532 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1533 PPDMBLKCACHEENTRY pEntry = NULL;
1534
1535 STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
1536
1537 RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1538 pEntry = (PPDMBLKCACHEENTRY)RTAvlrU64RangeGet(pBlkCache->pTree, off);
1539 if (pEntry)
1540 pdmBlkCacheEntryRef(pEntry);
1541 RTSemRWReleaseRead(pBlkCache->SemRWEntries);
1542
1543 STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
1544
1545 return pEntry;
1546}
1547
1548/**
1549 * Return the best fit cache entries for the given offset.
1550 *
1551 * @returns nothing.
1552 * @param pBlkCache The endpoint cache.
1553 * @param off The offset.
1554 * @param pEntryAbove Where to store the pointer to the best fit entry above the
1555 * the given offset. NULL if not required.
1556 */
1557static void pdmBlkCacheGetCacheBestFitEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off,
1558 PPDMBLKCACHEENTRY *ppEntryAbove)
1559{
1560 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1561
1562 STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
1563
1564 RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1565 if (ppEntryAbove)
1566 {
1567 *ppEntryAbove = (PPDMBLKCACHEENTRY)RTAvlrU64GetBestFit(pBlkCache->pTree, off, true /*fAbove*/);
1568 if (*ppEntryAbove)
1569 pdmBlkCacheEntryRef(*ppEntryAbove);
1570 }
1571
1572 RTSemRWReleaseRead(pBlkCache->SemRWEntries);
1573
1574 STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
1575}
1576
1577static void pdmBlkCacheInsertEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
1578{
1579 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1580
1581 STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache);
1582 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1583 bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
1584 AssertMsg(fInserted, ("Node was not inserted into tree\n"));
1585 STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache);
1586 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1587}
1588
1589/**
1590 * Allocates and initializes a new entry for the cache.
1591 * The entry has a reference count of 1.
1592 *
1593 * @returns Pointer to the new cache entry or NULL if out of memory.
1594 * @param pBlkCache The cache the entry belongs to.
1595 * @param off Start offset.
1596 * @param cbData Size of the cache entry.
1597 * @param pbBuffer Pointer to the buffer to use.
1598 * NULL if a new buffer should be allocated.
1599 * The buffer needs to have the same size of the entry.
1600 */
1601static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
1602 uint64_t off, size_t cbData, uint8_t *pbBuffer)
1603{
1604 PPDMBLKCACHEENTRY pEntryNew = (PPDMBLKCACHEENTRY)RTMemAllocZ(sizeof(PDMBLKCACHEENTRY));
1605
1606 if (RT_UNLIKELY(!pEntryNew))
1607 return NULL;
1608
1609 pEntryNew->Core.Key = off;
1610 pEntryNew->Core.KeyLast = off + cbData - 1;
1611 pEntryNew->pBlkCache = pBlkCache;
1612 pEntryNew->fFlags = 0;
1613 pEntryNew->cRefs = 1; /* We are using it now. */
1614 pEntryNew->pList = NULL;
1615 pEntryNew->cbData = cbData;
1616 pEntryNew->pWaitingHead = NULL;
1617 pEntryNew->pWaitingTail = NULL;
1618 if (pbBuffer)
1619 pEntryNew->pbData = pbBuffer;
1620 else
1621 pEntryNew->pbData = (uint8_t *)RTMemPageAlloc(cbData);
1622
1623 if (RT_UNLIKELY(!pEntryNew->pbData))
1624 {
1625 RTMemFree(pEntryNew);
1626 return NULL;
1627 }
1628
1629 return pEntryNew;
1630}
1631
1632/**
1633 * Checks that a set of flags is set/clear acquiring the R/W semaphore
1634 * in exclusive mode.
1635 *
1636 * @returns true if the flag in fSet is set and the one in fClear is clear.
1637 * false otherwise.
1638 * The R/W semaphore is only held if true is returned.
1639 *
1640 * @param pBlkCache The endpoint cache instance data.
1641 * @param pEntry The entry to check the flags for.
1642 * @param fSet The flag which is tested to be set.
1643 * @param fClear The flag which is tested to be clear.
1644 */
1645DECLINLINE(bool) pdmBlkCacheEntryFlagIsSetClearAcquireLock(PPDMBLKCACHE pBlkCache,
1646 PPDMBLKCACHEENTRY pEntry,
1647 uint32_t fSet, uint32_t fClear)
1648{
1649 uint32_t fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1650 bool fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1651
1652 if (fPassed)
1653 {
1654 /* Acquire the lock and check again because the completion callback might have raced us. */
1655 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1656
1657 fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1658 fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1659
1660 /* Drop the lock if we didn't passed the test. */
1661 if (!fPassed)
1662 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1663 }
1664
1665 return fPassed;
1666}
1667
1668/**
1669 * Adds a segment to the waiting list for a cache entry
1670 * which is currently in progress.
1671 *
1672 * @returns nothing.
1673 * @param pEntry The cache entry to add the segment to.
1674 * @param pSeg The segment to add.
1675 */
1676DECLINLINE(void) pdmBlkCacheEntryAddWaiter(PPDMBLKCACHEENTRY pEntry,
1677 PPDMBLKCACHEWAITER pWaiter)
1678{
1679 pWaiter->pNext = NULL;
1680
1681 if (pEntry->pWaitingHead)
1682 {
1683 AssertPtr(pEntry->pWaitingTail);
1684
1685 pEntry->pWaitingTail->pNext = pWaiter;
1686 pEntry->pWaitingTail = pWaiter;
1687 }
1688 else
1689 {
1690 Assert(!pEntry->pWaitingTail);
1691
1692 pEntry->pWaitingHead = pWaiter;
1693 pEntry->pWaitingTail = pWaiter;
1694 }
1695}
1696
1697/**
1698 * Add a buffer described by the I/O memory context
1699 * to the entry waiting for completion.
1700 *
1701 * @returns VBox status code.
1702 * @param pEntry The entry to add the buffer to.
1703 * @param pTask Task associated with the buffer.
1704 * @param pIoMemCtx The memory context to use.
1705 * @param offDiff Offset from the start of the buffer
1706 * in the entry.
1707 * @param cbData Amount of data to wait for onthis entry.
1708 * @param fWrite Flag whether the task waits because it wants to write
1709 * to the cache entry.
1710 */
1711static int pdmBlkCacheEntryWaitersAdd(PPDMBLKCACHEENTRY pEntry,
1712 PPDMBLKCACHEREQ pReq,
1713 PRTSGBUF pSgBuf, uint64_t offDiff,
1714 size_t cbData, bool fWrite)
1715{
1716 PPDMBLKCACHEWAITER pWaiter = (PPDMBLKCACHEWAITER)RTMemAllocZ(sizeof(PDMBLKCACHEWAITER));
1717 if (!pWaiter)
1718 return VERR_NO_MEMORY;
1719
1720 ASMAtomicIncU32(&pReq->cXfersPending);
1721 pWaiter->pReq = pReq;
1722 pWaiter->offCacheEntry = offDiff;
1723 pWaiter->cbTransfer = cbData;
1724 pWaiter->fWrite = fWrite;
1725 RTSgBufClone(&pWaiter->SgBuf, pSgBuf);
1726 RTSgBufAdvance(pSgBuf, cbData);
1727
1728 pdmBlkCacheEntryAddWaiter(pEntry, pWaiter);
1729
1730 return VINF_SUCCESS;
1731}
1732
1733/**
1734 * Calculate aligned offset and size for a new cache entry
1735 * which do not intersect with an already existing entry and the
1736 * file end.
1737 *
1738 * @returns The number of bytes the entry can hold of the requested amount
1739 * of byte.
1740 * @param pEndpoint The endpoint.
1741 * @param pBlkCache The endpoint cache.
1742 * @param off The start offset.
1743 * @param cb The number of bytes the entry needs to hold at least.
1744 * @param uAlignment Alignment of the boundary sizes.
1745 * @param poffAligned Where to store the aligned offset.
1746 * @param pcbAligned Where to store the aligned size of the entry.
1747 */
1748static size_t pdmBlkCacheEntryBoundariesCalc(PPDMBLKCACHE pBlkCache,
1749 uint64_t off, size_t cb,
1750 unsigned uAlignment,
1751 uint64_t *poffAligned, size_t *pcbAligned)
1752{
1753 size_t cbAligned;
1754 size_t cbInEntry = 0;
1755 uint64_t offAligned;
1756 PPDMBLKCACHEENTRY pEntryAbove = NULL;
1757
1758 /* Get the best fit entries around the offset */
1759 pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
1760
1761 /* Log the info */
1762 LogFlow(("%sest fit entry above off=%llu (BestFit=%llu BestFitEnd=%llu BestFitSize=%u)\n",
1763 pEntryAbove ? "B" : "No b",
1764 off,
1765 pEntryAbove ? pEntryAbove->Core.Key : 0,
1766 pEntryAbove ? pEntryAbove->Core.KeyLast : 0,
1767 pEntryAbove ? pEntryAbove->cbData : 0));
1768
1769 offAligned = off;
1770
1771 if ( pEntryAbove
1772 && off + cb > pEntryAbove->Core.Key)
1773 {
1774 cbInEntry = pEntryAbove->Core.Key - off;
1775 cbAligned = pEntryAbove->Core.Key - offAligned;
1776 }
1777 else
1778 {
1779 cbAligned = cb;
1780 cbInEntry = cb;
1781 }
1782
1783 /* A few sanity checks */
1784 AssertMsg(!pEntryAbove || (offAligned + cbAligned) <= pEntryAbove->Core.Key,
1785 ("Aligned size intersects with another cache entry\n"));
1786 Assert(cbInEntry <= cbAligned);
1787
1788 if (pEntryAbove)
1789 pdmBlkCacheEntryRelease(pEntryAbove);
1790
1791 LogFlow(("offAligned=%llu cbAligned=%u\n", offAligned, cbAligned));
1792
1793 *poffAligned = offAligned;
1794 *pcbAligned = cbAligned;
1795
1796 return cbInEntry;
1797}
1798
1799/**
1800 * Create a new cache entry evicting data from the cache if required.
1801 *
1802 * @returns Pointer to the new cache entry or NULL
1803 * if not enough bytes could be evicted from the cache.
1804 * @param pEndpoint The endpoint.
1805 * @param pBlkCache The endpoint cache.
1806 * @param off The offset.
1807 * @param cb Number of bytes the cache entry should have.
1808 * @param uAlignment Alignment the size of the entry should have.
1809 * @param pcbData Where to store the number of bytes the new
1810 * entry can hold. May be lower than actually requested
1811 * due to another entry intersecting the access range.
1812 */
1813static PPDMBLKCACHEENTRY pdmBlkCacheEntryCreate(PPDMBLKCACHE pBlkCache,
1814 uint64_t off, size_t cb,
1815 unsigned uAlignment,
1816 size_t *pcbData)
1817{
1818 uint64_t offStart = 0;
1819 size_t cbEntry = 0;
1820 PPDMBLKCACHEENTRY pEntryNew = NULL;
1821 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1822 uint8_t *pbBuffer = NULL;
1823
1824 *pcbData = pdmBlkCacheEntryBoundariesCalc(pBlkCache, off, cb, uAlignment,
1825 &offStart, &cbEntry);
1826
1827 pdmBlkCacheLockEnter(pCache);
1828 bool fEnough = pdmBlkCacheReclaim(pCache, cbEntry, true, &pbBuffer);
1829
1830 if (fEnough)
1831 {
1832 LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbEntry));
1833
1834 pEntryNew = pdmBlkCacheEntryAlloc(pBlkCache, offStart, cbEntry, pbBuffer);
1835 if (RT_LIKELY(pEntryNew))
1836 {
1837 pdmBlkCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
1838 pdmBlkCacheAdd(pCache, cbEntry);
1839 pdmBlkCacheLockLeave(pCache);
1840
1841 pdmBlkCacheInsertEntry(pBlkCache, pEntryNew);
1842
1843 AssertMsg( (off >= pEntryNew->Core.Key)
1844 && (off + *pcbData <= pEntryNew->Core.KeyLast + 1),
1845 ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
1846 off, pEntryNew->Core.Key));
1847 }
1848 else
1849 pdmBlkCacheLockLeave(pCache);
1850 }
1851 else
1852 pdmBlkCacheLockLeave(pCache);
1853
1854 return pEntryNew;
1855}
1856
1857static PPDMBLKCACHEREQ pdmBlkCacheReqAlloc(void *pvUser)
1858{
1859 PPDMBLKCACHEREQ pReq = (PPDMBLKCACHEREQ)RTMemAlloc(sizeof(PDMBLKCACHEREQ));
1860
1861 if (RT_LIKELY(pReq))
1862 {
1863 pReq->pvUser = pvUser;
1864 pReq->rcReq = VINF_SUCCESS;
1865 pReq->cXfersPending = 0;
1866 }
1867
1868 return pReq;
1869}
1870
1871static void pdmBlkCacheReqComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq)
1872{
1873 switch (pBlkCache->enmType)
1874 {
1875 case PDMBLKCACHETYPE_DEV:
1876 {
1877 pBlkCache->u.Dev.pfnXferComplete(pBlkCache->u.Dev.pDevIns,
1878 pReq->pvUser, pReq->rcReq);
1879 break;
1880 }
1881 case PDMBLKCACHETYPE_DRV:
1882 {
1883 pBlkCache->u.Drv.pfnXferComplete(pBlkCache->u.Drv.pDrvIns,
1884 pReq->pvUser, pReq->rcReq);
1885 break;
1886 }
1887 case PDMBLKCACHETYPE_USB:
1888 {
1889 pBlkCache->u.Usb.pfnXferComplete(pBlkCache->u.Usb.pUsbIns,
1890 pReq->pvUser, pReq->rcReq);
1891 break;
1892 }
1893 case PDMBLKCACHETYPE_INTERNAL:
1894 {
1895 pBlkCache->u.Int.pfnXferComplete(pBlkCache->u.Int.pvUser,
1896 pReq->pvUser, pReq->rcReq);
1897 break;
1898 }
1899 default:
1900 AssertMsgFailed(("Unknown block cache type!\n"));
1901 }
1902
1903 RTMemFree(pReq);
1904}
1905
1906static bool pdmBlkCacheReqUpdate(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
1907 int rcReq, bool fCallHandler)
1908{
1909 if (RT_FAILURE(rcReq))
1910 ASMAtomicCmpXchgS32(&pReq->rcReq, rcReq, VINF_SUCCESS);
1911
1912 AssertMsg(pReq->cXfersPending > 0, ("No transfers are pending for this request\n"));
1913 uint32_t cXfersPending = ASMAtomicDecU32(&pReq->cXfersPending);
1914
1915 if (!cXfersPending)
1916 {
1917 if (fCallHandler)
1918 pdmBlkCacheReqComplete(pBlkCache, pReq);
1919 else
1920 RTMemFree(pReq);
1921 return true;
1922 }
1923
1924 LogFlowFunc(("pReq=%#p cXfersPending=%u\n", pReq, cXfersPending));
1925 return false;
1926}
1927
1928VMMR3DECL(int) PDMR3BlkCacheRead(PPDMBLKCACHE pBlkCache, uint64_t off,
1929 PCRTSGBUF pcSgBuf, size_t cbRead, void *pvUser)
1930{
1931 int rc = VINF_SUCCESS;
1932 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1933 PPDMBLKCACHEENTRY pEntry;
1934 PPDMBLKCACHEREQ pReq;
1935
1936 LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbRead=%u pvUser=%#p\n",
1937 pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbRead, pvUser));
1938
1939 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
1940 AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
1941
1942 RTSGBUF SgBuf;
1943 RTSgBufClone(&SgBuf, pcSgBuf);
1944
1945 /* Allocate new request structure. */
1946 pReq = pdmBlkCacheReqAlloc(pvUser);
1947 if (RT_UNLIKELY(!pReq))
1948 return VERR_NO_MEMORY;
1949
1950 /* Increment data transfer counter to keep the request valid while we access it. */
1951 ASMAtomicIncU32(&pReq->cXfersPending);
1952
1953 while (cbRead)
1954 {
1955 size_t cbToRead;
1956
1957 pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
1958
1959 /*
1960 * If there is no entry we try to create a new one eviciting unused pages
1961 * if the cache is full. If this is not possible we will pass the request through
1962 * and skip the caching (all entries may be still in progress so they can't
1963 * be evicted)
1964 * If we have an entry it can be in one of the LRU lists where the entry
1965 * contains data (recently used or frequently used LRU) so we can just read
1966 * the data we need and put the entry at the head of the frequently used LRU list.
1967 * In case the entry is in one of the ghost lists it doesn't contain any data.
1968 * We have to fetch it again evicting pages from either T1 or T2 to make room.
1969 */
1970 if (pEntry)
1971 {
1972 uint64_t offDiff = off - pEntry->Core.Key;
1973
1974 AssertMsg(off >= pEntry->Core.Key,
1975 ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
1976 off, pEntry->Core.Key));
1977
1978 AssertPtr(pEntry->pList);
1979
1980 cbToRead = RT_MIN(pEntry->cbData - offDiff, cbRead);
1981
1982 AssertMsg(off + cbToRead <= pEntry->Core.Key + pEntry->Core.KeyLast + 1,
1983 ("Buffer of cache entry exceeded off=%llu cbToRead=%d\n",
1984 off, cbToRead));
1985
1986 cbRead -= cbToRead;
1987
1988 if (!cbRead)
1989 STAM_COUNTER_INC(&pCache->cHits);
1990 else
1991 STAM_COUNTER_INC(&pCache->cPartialHits);
1992
1993 STAM_COUNTER_ADD(&pCache->StatRead, cbToRead);
1994
1995 /* Ghost lists contain no data. */
1996 if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
1997 || (pEntry->pList == &pCache->LruFrequentlyUsed))
1998 {
1999 if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2000 PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
2001 PDMBLKCACHE_ENTRY_IS_DIRTY))
2002 {
2003 /* Entry didn't completed yet. Append to the list */
2004 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2005 &SgBuf, offDiff, cbToRead,
2006 false /* fWrite */);
2007 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2008 }
2009 else
2010 {
2011 /* Read as much as we can from the entry. */
2012 RTSgBufCopyFromBuf(&SgBuf, pEntry->pbData + offDiff, cbToRead);
2013 }
2014
2015 /* Move this entry to the top position */
2016 if (pEntry->pList == &pCache->LruFrequentlyUsed)
2017 {
2018 pdmBlkCacheLockEnter(pCache);
2019 pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2020 pdmBlkCacheLockLeave(pCache);
2021 }
2022 /* Release the entry */
2023 pdmBlkCacheEntryRelease(pEntry);
2024 }
2025 else
2026 {
2027 uint8_t *pbBuffer = NULL;
2028
2029 LogFlow(("Fetching data for ghost entry %#p from file\n", pEntry));
2030
2031 pdmBlkCacheLockEnter(pCache);
2032 pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
2033 bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
2034
2035 /* Move the entry to Am and fetch it to the cache. */
2036 if (fEnough)
2037 {
2038 pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2039 pdmBlkCacheAdd(pCache, pEntry->cbData);
2040 pdmBlkCacheLockLeave(pCache);
2041
2042 if (pbBuffer)
2043 pEntry->pbData = pbBuffer;
2044 else
2045 pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
2046 AssertPtr(pEntry->pbData);
2047
2048 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2049 &SgBuf, offDiff, cbToRead,
2050 false /* fWrite */);
2051 pdmBlkCacheEntryReadFromMedium(pEntry);
2052 /* Release the entry */
2053 pdmBlkCacheEntryRelease(pEntry);
2054 }
2055 else
2056 {
2057 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2058 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2059 RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
2060 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2061 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2062
2063 pdmBlkCacheLockLeave(pCache);
2064
2065 RTMemFree(pEntry);
2066
2067 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2068 &SgBuf, off, cbToRead,
2069 PDMBLKCACHEXFERDIR_READ);
2070 }
2071 }
2072 }
2073 else
2074 {
2075#ifdef VBOX_WITH_IO_READ_CACHE
2076 /* No entry found for this offset. Create a new entry and fetch the data to the cache. */
2077 PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
2078 off, cbRead,
2079 PAGE_SIZE,
2080 &cbToRead);
2081
2082 cbRead -= cbToRead;
2083
2084 if (pEntryNew)
2085 {
2086 if (!cbRead)
2087 STAM_COUNTER_INC(&pCache->cMisses);
2088 else
2089 STAM_COUNTER_INC(&pCache->cPartialHits);
2090
2091 pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
2092 &SgBuf,
2093 off - pEntryNew->Core.Key,
2094 cbToRead,
2095 false /* fWrite */);
2096 pdmBlkCacheEntryReadFromMedium(pEntryNew);
2097 pdmBlkCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
2098 }
2099 else
2100 {
2101 /*
2102 * There is not enough free space in the cache.
2103 * Pass the request directly to the I/O manager.
2104 */
2105 LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToRead));
2106
2107 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2108 &SgBuf, off, cbToRead,
2109 PDMBLKCACHEXFERDIR_READ);
2110 }
2111#else
2112 /* Clip read size if necessary. */
2113 PPDMBLKCACHEENTRY pEntryAbove;
2114 pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
2115
2116 if (pEntryAbove)
2117 {
2118 if (off + cbRead > pEntryAbove->Core.Key)
2119 cbToRead = pEntryAbove->Core.Key - off;
2120 else
2121 cbToRead = cbRead;
2122
2123 pdmBlkCacheEntryRelease(pEntryAbove);
2124 }
2125 else
2126 cbToRead = cbRead;
2127
2128 cbRead -= cbToRead;
2129 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2130 &SgBuf, off, cbToRead,
2131 PDMBLKCACHEXFERDIR_READ);
2132#endif
2133 }
2134 off += cbToRead;
2135 }
2136
2137 if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, false))
2138 rc = VINF_AIO_TASK_PENDING;
2139
2140 LogFlowFunc((": Leave rc=%Rrc\n", rc));
2141
2142 return rc;
2143}
2144
2145VMMR3DECL(int) PDMR3BlkCacheWrite(PPDMBLKCACHE pBlkCache, uint64_t off,
2146 PCRTSGBUF pcSgBuf, size_t cbWrite, void *pvUser)
2147{
2148 int rc = VINF_SUCCESS;
2149 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
2150 PPDMBLKCACHEENTRY pEntry;
2151 PPDMBLKCACHEREQ pReq;
2152
2153 LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbWrite=%u pvUser=%#p\n",
2154 pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbWrite, pvUser));
2155
2156 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
2157 AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
2158
2159 RTSGBUF SgBuf;
2160 RTSgBufClone(&SgBuf, pcSgBuf);
2161
2162 /* Allocate new request structure. */
2163 pReq = pdmBlkCacheReqAlloc(pvUser);
2164 if (RT_UNLIKELY(!pReq))
2165 return VERR_NO_MEMORY;
2166
2167 /* Increment data transfer counter to keep the request valid while we access it. */
2168 ASMAtomicIncU32(&pReq->cXfersPending);
2169
2170 while (cbWrite)
2171 {
2172 size_t cbToWrite;
2173
2174 pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
2175
2176 if (pEntry)
2177 {
2178 /* Write the data into the entry and mark it as dirty */
2179 AssertPtr(pEntry->pList);
2180
2181 uint64_t offDiff = off - pEntry->Core.Key;
2182
2183 AssertMsg(off >= pEntry->Core.Key,
2184 ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
2185 off, pEntry->Core.Key));
2186
2187 cbToWrite = RT_MIN(pEntry->cbData - offDiff, cbWrite);
2188 cbWrite -= cbToWrite;
2189
2190 if (!cbWrite)
2191 STAM_COUNTER_INC(&pCache->cHits);
2192 else
2193 STAM_COUNTER_INC(&pCache->cPartialHits);
2194
2195 STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
2196
2197 /* Ghost lists contain no data. */
2198 if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
2199 || (pEntry->pList == &pCache->LruFrequentlyUsed))
2200 {
2201 /* Check if the entry is dirty. */
2202 if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2203 PDMBLKCACHE_ENTRY_IS_DIRTY,
2204 0))
2205 {
2206 /* If it is already dirty but not in progress just update the data. */
2207 if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS))
2208 {
2209 RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff,
2210 cbToWrite);
2211 }
2212 else
2213 {
2214 /* The data isn't written to the file yet */
2215 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2216 &SgBuf, offDiff, cbToWrite,
2217 true /* fWrite */);
2218 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2219 }
2220
2221 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2222 }
2223 else /* Dirty bit not set */
2224 {
2225 /*
2226 * Check if a read is in progress for this entry.
2227 * We have to defer processing in that case.
2228 */
2229 if(pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2230 PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
2231 0))
2232 {
2233 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2234 &SgBuf, offDiff, cbToWrite,
2235 true /* fWrite */);
2236 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2237 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2238 }
2239 else /* I/O in progress flag not set */
2240 {
2241 /* Write as much as we can into the entry and update the file. */
2242 RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff, cbToWrite);
2243
2244 bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
2245 if (fCommit)
2246 pdmBlkCacheCommitDirtyEntries(pCache);
2247 }
2248 } /* Dirty bit not set */
2249
2250 /* Move this entry to the top position */
2251 if (pEntry->pList == &pCache->LruFrequentlyUsed)
2252 {
2253 pdmBlkCacheLockEnter(pCache);
2254 pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2255 pdmBlkCacheLockLeave(pCache);
2256 }
2257
2258 pdmBlkCacheEntryRelease(pEntry);
2259 }
2260 else /* Entry is on the ghost list */
2261 {
2262 uint8_t *pbBuffer = NULL;
2263
2264 pdmBlkCacheLockEnter(pCache);
2265 pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
2266 bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
2267
2268 if (fEnough)
2269 {
2270 /* Move the entry to Am and fetch it to the cache. */
2271 pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2272 pdmBlkCacheAdd(pCache, pEntry->cbData);
2273 pdmBlkCacheLockLeave(pCache);
2274
2275 if (pbBuffer)
2276 pEntry->pbData = pbBuffer;
2277 else
2278 pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
2279 AssertPtr(pEntry->pbData);
2280
2281 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2282 &SgBuf, offDiff, cbToWrite,
2283 true /* fWrite */);
2284 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2285 pdmBlkCacheEntryReadFromMedium(pEntry);
2286
2287 /* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
2288 pdmBlkCacheEntryRelease(pEntry);
2289 }
2290 else
2291 {
2292 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2293 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2294 RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
2295 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2296 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2297
2298 pdmBlkCacheLockLeave(pCache);
2299
2300 RTMemFree(pEntry);
2301 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2302 &SgBuf, off, cbToWrite,
2303 PDMBLKCACHEXFERDIR_WRITE);
2304 }
2305 }
2306 }
2307 else /* No entry found */
2308 {
2309 /*
2310 * No entry found. Try to create a new cache entry to store the data in and if that fails
2311 * write directly to the file.
2312 */
2313 PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
2314 off, cbWrite,
2315 512, &cbToWrite);
2316
2317 cbWrite -= cbToWrite;
2318
2319 if (pEntryNew)
2320 {
2321 uint64_t offDiff = off - pEntryNew->Core.Key;
2322
2323 STAM_COUNTER_INC(&pCache->cHits);
2324
2325 /*
2326 * Check if it is possible to just write the data without waiting
2327 * for it to get fetched first.
2328 */
2329 if (!offDiff && pEntryNew->cbData == cbToWrite)
2330 {
2331 RTSgBufCopyToBuf(&SgBuf, pEntryNew->pbData, cbToWrite);
2332
2333 bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntryNew);
2334 if (fCommit)
2335 pdmBlkCacheCommitDirtyEntries(pCache);
2336 STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
2337 }
2338 else
2339 {
2340 /* Defer the write and fetch the data from the endpoint. */
2341 pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
2342 &SgBuf, offDiff, cbToWrite,
2343 true /* fWrite */);
2344 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2345 pdmBlkCacheEntryReadFromMedium(pEntryNew);
2346 }
2347
2348 pdmBlkCacheEntryRelease(pEntryNew);
2349 }
2350 else
2351 {
2352 /*
2353 * There is not enough free space in the cache.
2354 * Pass the request directly to the I/O manager.
2355 */
2356 LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToWrite));
2357
2358 STAM_COUNTER_INC(&pCache->cMisses);
2359
2360 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2361 &SgBuf, off, cbToWrite,
2362 PDMBLKCACHEXFERDIR_WRITE);
2363 }
2364 }
2365
2366 off += cbToWrite;
2367 }
2368
2369 if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, false))
2370 rc = VINF_AIO_TASK_PENDING;
2371
2372 LogFlowFunc((": Leave rc=%Rrc\n", rc));
2373
2374 return rc;
2375}
2376
2377VMMR3DECL(int) PDMR3BlkCacheFlush(PPDMBLKCACHE pBlkCache, void *pvUser)
2378{
2379 int rc = VINF_SUCCESS;
2380 PPDMBLKCACHEREQ pReq;
2381
2382 LogFlowFunc((": pBlkCache=%#p{%s}\n", pBlkCache, pBlkCache->pszId));
2383
2384 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
2385 AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
2386
2387 /* Commit dirty entries in the cache. */
2388 pdmBlkCacheCommit(pBlkCache);
2389
2390 /* Allocate new request structure. */
2391 pReq = pdmBlkCacheReqAlloc(pvUser);
2392 if (RT_UNLIKELY(!pReq))
2393 return VERR_NO_MEMORY;
2394
2395 rc = pdmBlkCacheRequestPassthrough(pBlkCache, pReq, NULL, 0, 0,
2396 PDMBLKCACHEXFERDIR_FLUSH);
2397 AssertRC(rc);
2398
2399 LogFlowFunc((": Leave rc=%Rrc\n", rc));
2400 return VINF_AIO_TASK_PENDING;
2401}
2402
2403/**
2404 * Completes a task segment freeing all resources and completes the task handle
2405 * if everything was transferred.
2406 *
2407 * @returns Next task segment handle.
2408 * @param pTaskSeg Task segment to complete.
2409 * @param rc Status code to set.
2410 */
2411static PPDMBLKCACHEWAITER pdmBlkCacheWaiterComplete(PPDMBLKCACHE pBlkCache,
2412 PPDMBLKCACHEWAITER pWaiter,
2413 int rc)
2414{
2415 PPDMBLKCACHEWAITER pNext = pWaiter->pNext;
2416 PPDMBLKCACHEREQ pReq = pWaiter->pReq;
2417
2418 pdmBlkCacheReqUpdate(pBlkCache, pWaiter->pReq, rc, true);
2419
2420 RTMemFree(pWaiter);
2421
2422 return pNext;
2423}
2424
2425static void pdmBlkCacheIoXferCompleteEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
2426{
2427 PPDMBLKCACHEENTRY pEntry = hIoXfer->pEntry;
2428 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
2429
2430 /* Reference the entry now as we are clearing the I/O in progress flag
2431 * which protected the entry till now. */
2432 pdmBlkCacheEntryRef(pEntry);
2433
2434 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2435 pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
2436
2437 /* Process waiting segment list. The data in entry might have changed in-between. */
2438 bool fDirty = false;
2439 PPDMBLKCACHEWAITER pComplete = pEntry->pWaitingHead;
2440 PPDMBLKCACHEWAITER pCurr = pComplete;
2441
2442 AssertMsg((pCurr && pEntry->pWaitingTail) || (!pCurr && !pEntry->pWaitingTail),
2443 ("The list tail was not updated correctly\n"));
2444 pEntry->pWaitingTail = NULL;
2445 pEntry->pWaitingHead = NULL;
2446
2447 if (hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_WRITE)
2448 {
2449 /*
2450 * An error here is difficult to handle as the original request completed already.
2451 * The error is logged for now and the VM is paused.
2452 * If the user continues the entry is written again in the hope
2453 * the user fixed the problem and the next write succeeds.
2454 */
2455 if (RT_FAILURE(rcIoXfer))
2456 {
2457 LogRel(("I/O cache: Error while writing entry at offset %llu (%u bytes) to medium \"%s\" (rc=%Rrc)\n",
2458 pEntry->Core.Key, pEntry->cbData, pBlkCache->pszId, rcIoXfer));
2459
2460 if (!ASMAtomicXchgBool(&pCache->fIoErrorVmSuspended, true))
2461 {
2462 int rc = VMSetRuntimeError(pCache->pVM, VMSETRTERR_FLAGS_SUSPEND | VMSETRTERR_FLAGS_NO_WAIT, "BLKCACHE_IOERR",
2463 N_("The I/O cache encountered an error while updating data in medium \"%s\" (rc=%Rrc). "
2464 "Make sure there is enough free space on the disk and that the disk is working properly. "
2465 "Operation can be resumed afterwards"),
2466 pBlkCache->pszId, rcIoXfer);
2467 AssertRC(rc);
2468 }
2469
2470 /* Mark the entry as dirty again to get it added to the list later on. */
2471 fDirty = true;
2472 }
2473
2474 pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IS_DIRTY;
2475
2476 while (pCurr)
2477 {
2478 AssertMsg(pCurr->fWrite, ("Completed write entries should never have read tasks attached\n"));
2479
2480 RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2481 fDirty = true;
2482 pCurr = pCurr->pNext;
2483 }
2484 }
2485 else
2486 {
2487 AssertMsg(hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_READ, ("Invalid transfer type\n"));
2488 AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY),
2489 ("Invalid flags set\n"));
2490
2491 while (pCurr)
2492 {
2493 if (pCurr->fWrite)
2494 {
2495 RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2496 fDirty = true;
2497 }
2498 else
2499 RTSgBufCopyFromBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2500
2501 pCurr = pCurr->pNext;
2502 }
2503 }
2504
2505 bool fCommit = false;
2506 if (fDirty)
2507 fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
2508
2509 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2510
2511 /* Dereference so that it isn't protected anymore except we issued anyother write for it. */
2512 pdmBlkCacheEntryRelease(pEntry);
2513
2514 if (fCommit)
2515 pdmBlkCacheCommitDirtyEntries(pCache);
2516
2517 /* Complete waiters now. */
2518 while (pComplete)
2519 pComplete = pdmBlkCacheWaiterComplete(pBlkCache, pComplete, rcIoXfer);
2520}
2521
2522VMMR3DECL(void) PDMR3BlkCacheIoXferComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
2523{
2524 LogFlowFunc(("pBlkCache=%#p hIoXfer=%#p rcIoXfer=%Rrc\n", pBlkCache, hIoXfer, rcIoXfer));
2525
2526 if (hIoXfer->fIoCache)
2527 pdmBlkCacheIoXferCompleteEntry(pBlkCache, hIoXfer, rcIoXfer);
2528 else
2529 pdmBlkCacheReqUpdate(pBlkCache, hIoXfer->pReq, rcIoXfer, true);
2530 RTMemFree(hIoXfer);
2531}
2532
2533/**
2534 * Callback for the AVL do with all routine. Waits for a cachen entry to finish any pending I/O.
2535 *
2536 * @returns IPRT status code.
2537 * @param pNode The node to destroy.
2538 * @param pvUser Opaque user data.
2539 */
2540static int pdmBlkCacheEntryQuiesce(PAVLRU64NODECORE pNode, void *pvUser)
2541{
2542 PPDMBLKCACHEENTRY pEntry = (PPDMBLKCACHEENTRY)pNode;
2543 PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
2544
2545 while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
2546 {
2547 /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
2548 pdmBlkCacheEntryRef(pEntry);
2549 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2550
2551 RTThreadSleep(1);
2552
2553 /* Re-enter all locks and drop the reference. */
2554 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2555 pdmBlkCacheEntryRelease(pEntry);
2556 }
2557
2558 AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
2559 ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
2560
2561 return VINF_SUCCESS;
2562}
2563
2564VMMR3DECL(int) PDMR3BlkCacheSuspend(PPDMBLKCACHE pBlkCache)
2565{
2566 int rc = VINF_SUCCESS;
2567 LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
2568
2569 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
2570
2571 ASMAtomicXchgBool(&pBlkCache->fSuspended, true);
2572
2573 /* Wait for all I/O to complete. */
2574 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2575 rc = RTAvlrU64DoWithAll(pBlkCache->pTree, true, pdmBlkCacheEntryQuiesce, NULL);
2576 AssertRC(rc);
2577 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2578
2579 return rc;
2580}
2581
2582VMMR3DECL(int) PDMR3BlkCacheResume(PPDMBLKCACHE pBlkCache)
2583{
2584 LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
2585
2586 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
2587
2588 ASMAtomicXchgBool(&pBlkCache->fSuspended, false);
2589
2590 return VINF_SUCCESS;
2591}
2592
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette