VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 50653

Last change on this file since 50653 was 49893, checked in by vboxsync, 10 years ago

MSR rewrite: initial hacking - half disabled.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 47.3 KB
Line 
1/* $Id: MMAllHyper.cpp 49893 2013-12-13 00:40:20Z vboxsync $ */
2/** @file
3 * MM - Memory Manager - Hypervisor Memory Area, All Contexts.
4 */
5
6/*
7 * Copyright (C) 2006-2012 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
23#include <VBox/vmm/mm.h>
24#include <VBox/vmm/stam.h>
25#include "MMInternal.h"
26#include <VBox/vmm/vm.h>
27
28#include <VBox/err.h>
29#include <VBox/param.h>
30#include <iprt/assert.h>
31#include <VBox/log.h>
32#include <iprt/asm.h>
33#include <iprt/string.h>
34
35
36/*******************************************************************************
37* Defined Constants And Macros *
38*******************************************************************************/
39#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
40#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
41#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
42#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
43
44#define ASSERT_OFFPREV(pHeap, pChunk) \
45 do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
46 Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
47 AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
48 || (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
49 ("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
50 } while (0)
51
52#define ASSERT_OFFNEXT(pHeap, pChunk) \
53 do { ASSERT_ALIGN((pChunk)->offNext); \
54 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
55 } while (0)
56
57#define ASSERT_OFFHEAP(pHeap, pChunk) \
58 do { Assert((pChunk)->offHeap); \
59 AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
60 ("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
61 Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
62 } while (0)
63
64#ifdef VBOX_WITH_STATISTICS
65#define ASSERT_OFFSTAT(pHeap, pChunk) \
66 do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
67 Assert(!(pChunk)->offStat); \
68 else if ((pChunk)->offStat) \
69 { \
70 Assert((pChunk)->offStat); \
71 AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
72 uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
73 AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
74 ("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
75 } \
76 } while (0)
77#else
78#define ASSERT_OFFSTAT(pHeap, pChunk) \
79 do { Assert(!(pChunk)->offStat); \
80 } while (0)
81#endif
82
83#define ASSERT_CHUNK(pHeap, pChunk) \
84 do { ASSERT_OFFNEXT(pHeap, pChunk); \
85 ASSERT_OFFPREV(pHeap, pChunk); \
86 ASSERT_OFFHEAP(pHeap, pChunk); \
87 ASSERT_OFFSTAT(pHeap, pChunk); \
88 } while (0)
89#define ASSERT_CHUNK_USED(pHeap, pChunk) \
90 do { ASSERT_OFFNEXT(pHeap, pChunk); \
91 ASSERT_OFFPREV(pHeap, pChunk); \
92 Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
93 } while (0)
94
95#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
96 do { ASSERT_ALIGN((pChunk)->offPrev); \
97 ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
98 Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) || !(pChunk)->offPrev); \
99 AssertMsg( (pChunk)->offPrev \
100 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
101 ("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
102 (pHeap)->offFreeHead)); \
103 } while (0)
104
105#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
106 do { ASSERT_ALIGN((pChunk)->offNext); \
107 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
108 Assert((pChunk)->offNext != (pChunk)->core.offNext || !(pChunk)->offNext); \
109 AssertMsg( (pChunk)->offNext \
110 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
111 ("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
112 (pHeap)->offFreeTail)); \
113 } while (0)
114
115#define ASSERT_FREE_CB(pHeap, pChunk) \
116 do { ASSERT_ALIGN((pChunk)->cb); \
117 Assert((pChunk)->cb > 0); \
118 if ((pChunk)->core.offNext) \
119 AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
120 ("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
121 else \
122 ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
123 } while (0)
124
125#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
126 do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
127 Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
128 ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
129 ASSERT_FREE_OFFPREV(pHeap, pChunk); \
130 ASSERT_FREE_CB(pHeap, pChunk); \
131 } while (0)
132
133
134/*******************************************************************************
135* Internal Functions *
136*******************************************************************************/
137static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
138static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
139#ifdef VBOX_WITH_STATISTICS
140static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
141#ifdef IN_RING3
142static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
143#endif
144#endif
145static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
146#ifdef MMHYPER_HEAP_STRICT
147static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
148#endif
149
150
151
152/**
153 * Locks the hypervisor heap.
154 * This might call back to Ring-3 in order to deal with lock contention in GC and R3.
155 *
156 * @param pVM Pointer to the VM.
157 */
158static int mmHyperLock(PVM pVM)
159{
160 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
161
162#ifdef IN_RING3
163 if (!PDMCritSectIsInitialized(&pHeap->Lock))
164 return VINF_SUCCESS; /* early init */
165#else
166 Assert(PDMCritSectIsInitialized(&pHeap->Lock));
167#endif
168 int rc = PDMCritSectEnter(&pHeap->Lock, VERR_SEM_BUSY);
169#if defined(IN_RC) || defined(IN_RING0)
170 if (rc == VERR_SEM_BUSY)
171 rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_MMHYPER_LOCK, 0);
172#endif
173 AssertRC(rc);
174 return rc;
175}
176
177
178/**
179 * Unlocks the hypervisor heap.
180 *
181 * @param pVM Pointer to the VM.
182 */
183static void mmHyperUnlock(PVM pVM)
184{
185 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
186
187#ifdef IN_RING3
188 if (!PDMCritSectIsInitialized(&pHeap->Lock))
189 return; /* early init */
190#endif
191 Assert(PDMCritSectIsInitialized(&pHeap->Lock));
192 PDMCritSectLeave(&pHeap->Lock);
193}
194
195/**
196 * Allocates memory in the Hypervisor (RC VMM) area.
197 * The returned memory is of course zeroed.
198 *
199 * @returns VBox status code.
200 * @param pVM Pointer to the VM.
201 * @param cb Number of bytes to allocate.
202 * @param uAlignment Required memory alignment in bytes.
203 * Values are 0,8,16,32,64 and PAGE_SIZE.
204 * 0 -> default alignment, i.e. 8 bytes.
205 * @param enmTag The statistics tag.
206 * @param ppv Where to store the address to the allocated
207 * memory.
208 */
209static int mmHyperAllocInternal(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
210{
211 AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
212
213 /*
214 * Validate input and adjust it to reasonable values.
215 */
216 if (!uAlignment || uAlignment < MMHYPER_HEAP_ALIGN_MIN)
217 uAlignment = MMHYPER_HEAP_ALIGN_MIN;
218 uint32_t cbAligned;
219 switch (uAlignment)
220 {
221 case 8:
222 case 16:
223 case 32:
224 case 64:
225 cbAligned = RT_ALIGN_32(cb, MMHYPER_HEAP_ALIGN_MIN);
226 if (!cbAligned || cbAligned < cb)
227 {
228 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
229 AssertMsgFailed(("Nice try.\n"));
230 return VERR_INVALID_PARAMETER;
231 }
232 break;
233
234 case PAGE_SIZE:
235 AssertMsg(RT_ALIGN_32(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
236 cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
237 if (!cbAligned)
238 {
239 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
240 AssertMsgFailed(("Nice try.\n"));
241 return VERR_INVALID_PARAMETER;
242 }
243 break;
244
245 default:
246 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
247 AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
248 return VERR_INVALID_PARAMETER;
249 }
250
251
252 /*
253 * Get heap and statisticsStatistics.
254 */
255 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
256#ifdef VBOX_WITH_STATISTICS
257 PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
258 if (!pStat)
259 {
260 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
261 AssertMsgFailed(("Failed to allocate statistics!\n"));
262 return VERR_MM_HYPER_NO_MEMORY;
263 }
264#endif
265 if (uAlignment < PAGE_SIZE)
266 {
267 /*
268 * Allocate a chunk.
269 */
270 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
271 if (pChunk)
272 {
273#ifdef VBOX_WITH_STATISTICS
274 const uint32_t cbChunk = pChunk->offNext
275 ? pChunk->offNext
276 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
277 pStat->cbAllocated += (uint32_t)cbChunk;
278 pStat->cbCurAllocated += (uint32_t)cbChunk;
279 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
280 pStat->cbMaxAllocated = pStat->cbCurAllocated;
281 pStat->cAllocations++;
282 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
283#else
284 pChunk->offStat = 0;
285#endif
286 void *pv = pChunk + 1;
287 *ppv = pv;
288 ASMMemZero32(pv, cbAligned);
289 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
290 return VINF_SUCCESS;
291 }
292 }
293 else
294 {
295 /*
296 * Allocate page aligned memory.
297 */
298 void *pv = mmHyperAllocPages(pHeap, cbAligned);
299 if (pv)
300 {
301#ifdef VBOX_WITH_STATISTICS
302 pStat->cbAllocated += cbAligned;
303 pStat->cbCurAllocated += cbAligned;
304 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
305 pStat->cbMaxAllocated = pStat->cbCurAllocated;
306 pStat->cAllocations++;
307#endif
308 *ppv = pv;
309 /* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPR3PageAlloc zeros it. */
310 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
311 return VINF_SUCCESS;
312 }
313 }
314
315#ifdef VBOX_WITH_STATISTICS
316 pStat->cAllocations++;
317 pStat->cFailures++;
318#endif
319 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
320 AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
321 return VERR_MM_HYPER_NO_MEMORY;
322}
323
324
325/**
326 * Wrapper for mmHyperAllocInternal
327 */
328VMMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
329{
330 int rc = mmHyperLock(pVM);
331 AssertRCReturn(rc, rc);
332
333 LogFlow(("MMHyperAlloc %x align=%x tag=%s\n", cb, uAlignment, mmGetTagName(enmTag)));
334
335 rc = mmHyperAllocInternal(pVM, cb, uAlignment, enmTag, ppv);
336
337 mmHyperUnlock(pVM);
338 return rc;
339}
340
341
342/**
343 * Duplicates a block of memory.
344 */
345VMMDECL(int) MMHyperDupMem(PVM pVM, const void *pvSrc, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
346{
347 int rc = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
348 if (RT_SUCCESS(rc))
349 memcpy(*ppv, pvSrc, cb);
350 return rc;
351}
352
353
354/**
355 * Allocates a chunk of memory from the specified heap.
356 * The caller validates the parameters of this request.
357 *
358 * @returns Pointer to the allocated chunk.
359 * @returns NULL on failure.
360 * @param pHeap The heap.
361 * @param cb Size of the memory block to allocate.
362 * @param uAlignment The alignment specifications for the allocated block.
363 * @internal
364 */
365static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
366{
367 Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
368#ifdef MMHYPER_HEAP_STRICT
369 mmHyperHeapCheck(pHeap);
370#endif
371#ifdef MMHYPER_HEAP_STRICT_FENCE
372 uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
373 cb += cbFence;
374#endif
375
376 /*
377 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
378 */
379 if (pHeap->offFreeHead == NIL_OFFSET)
380 return NULL;
381
382 /*
383 * Small alignments - from the front of the heap.
384 *
385 * Must split off free chunks at the end to prevent messing up the
386 * last free node which we take the page aligned memory from the top of.
387 */
388 PMMHYPERCHUNK pRet = NULL;
389 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
390 while (pFree)
391 {
392 ASSERT_CHUNK_FREE(pHeap, pFree);
393 if (pFree->cb >= cb)
394 {
395 unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
396 if (offAlign)
397 offAlign = uAlignment - offAlign;
398 if (!offAlign || pFree->cb - offAlign >= cb)
399 {
400 Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
401
402 /*
403 * Adjust the node in front.
404 * Because of multiple alignments we need to special case allocation of the first block.
405 */
406 if (offAlign)
407 {
408 MMHYPERCHUNKFREE Free = *pFree;
409 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
410 {
411 /* just add a bit of memory to it. */
412 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
413 pPrev->core.offNext += offAlign;
414 AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
415 Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
416 }
417 else
418 {
419 /* make new head node, mark it USED for simplicity. */
420 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
421 Assert(pPrev == &pFree->core);
422 pPrev->offPrev = 0;
423 MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
424 pPrev->offNext = offAlign;
425 Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
426
427 }
428 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
429 pHeap->cbFree -= offAlign;
430
431 /* Recreate pFree node and adjusting everything... */
432 pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
433 *pFree = Free;
434
435 pFree->cb -= offAlign;
436 if (pFree->core.offNext)
437 {
438 pFree->core.offNext -= offAlign;
439 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
440 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
441 ASSERT_CHUNK(pHeap, pNext);
442 }
443 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
444 MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
445
446 if (pFree->offNext)
447 {
448 pFree->offNext -= offAlign;
449 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
450 pNext->offPrev = -(int32_t)pFree->offNext;
451 ASSERT_CHUNK_FREE(pHeap, pNext);
452 }
453 else
454 pHeap->offFreeTail += offAlign;
455 if (pFree->offPrev)
456 {
457 pFree->offPrev -= offAlign;
458 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
459 pPrev->offNext = -pFree->offPrev;
460 ASSERT_CHUNK_FREE(pHeap, pPrev);
461 }
462 else
463 pHeap->offFreeHead += offAlign;
464 pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
465 pFree->core.offStat = 0;
466 ASSERT_CHUNK_FREE(pHeap, pFree);
467 Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
468 }
469
470 /*
471 * Split off a new FREE chunk?
472 */
473 if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
474 {
475 /*
476 * Move the FREE chunk up to make room for the new USED chunk.
477 */
478 const int off = cb + sizeof(MMHYPERCHUNK);
479 PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
480 *pNew = *pFree;
481 pNew->cb -= off;
482 if (pNew->core.offNext)
483 {
484 pNew->core.offNext -= off;
485 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
486 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
487 ASSERT_CHUNK(pHeap, pNext);
488 }
489 pNew->core.offPrev = -off;
490 MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
491
492 if (pNew->offNext)
493 {
494 pNew->offNext -= off;
495 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
496 pNext->offPrev = -(int32_t)pNew->offNext;
497 ASSERT_CHUNK_FREE(pHeap, pNext);
498 }
499 else
500 pHeap->offFreeTail += off;
501 if (pNew->offPrev)
502 {
503 pNew->offPrev -= off;
504 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
505 pPrev->offNext = -pNew->offPrev;
506 ASSERT_CHUNK_FREE(pHeap, pPrev);
507 }
508 else
509 pHeap->offFreeHead += off;
510 pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
511 pNew->core.offStat = 0;
512 ASSERT_CHUNK_FREE(pHeap, pNew);
513
514 /*
515 * Update the old FREE node making it a USED node.
516 */
517 pFree->core.offNext = off;
518 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
519
520
521 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
522 pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
523 pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
524 pRet = &pFree->core;
525 ASSERT_CHUNK(pHeap, &pFree->core);
526 Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
527 }
528 else
529 {
530 /*
531 * Link out of free list.
532 */
533 if (pFree->offNext)
534 {
535 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
536 if (pFree->offPrev)
537 {
538 pNext->offPrev += pFree->offPrev;
539 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
540 pPrev->offNext += pFree->offNext;
541 ASSERT_CHUNK_FREE(pHeap, pPrev);
542 }
543 else
544 {
545 pHeap->offFreeHead += pFree->offNext;
546 pNext->offPrev = 0;
547 }
548 ASSERT_CHUNK_FREE(pHeap, pNext);
549 }
550 else
551 {
552 if (pFree->offPrev)
553 {
554 pHeap->offFreeTail += pFree->offPrev;
555 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
556 pPrev->offNext = 0;
557 ASSERT_CHUNK_FREE(pHeap, pPrev);
558 }
559 else
560 {
561 pHeap->offFreeHead = NIL_OFFSET;
562 pHeap->offFreeTail = NIL_OFFSET;
563 }
564 }
565
566 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
567 pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
568 pHeap->cbFree -= pFree->cb;
569 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
570 pRet = &pFree->core;
571 ASSERT_CHUNK(pHeap, &pFree->core);
572 Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
573 }
574 Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
575 break;
576 }
577 }
578
579 /* next */
580 pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
581 }
582
583#ifdef MMHYPER_HEAP_STRICT_FENCE
584 uint32_t *pu32End = (uint32_t *)((uint8_t *)(pRet + 1) + cb);
585 uint32_t *pu32EndReal = pRet->offNext
586 ? (uint32_t *)((uint8_t *)pRet + pRet->offNext)
587 : (uint32_t *)(pHeap->CTX_SUFF(pbHeap) + pHeap->cbHeap);
588 cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
589 ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
590 pu32EndReal[-1] = cbFence;
591#endif
592#ifdef MMHYPER_HEAP_STRICT
593 mmHyperHeapCheck(pHeap);
594#endif
595 return pRet;
596}
597
598
599/**
600 * Allocates one or more pages of memory from the specified heap.
601 * The caller validates the parameters of this request.
602 *
603 * @returns Pointer to the allocated chunk.
604 * @returns NULL on failure.
605 * @param pHeap The heap.
606 * @param cb Size of the memory block to allocate.
607 * @internal
608 */
609static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
610{
611 Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
612
613#ifdef MMHYPER_HEAP_STRICT
614 mmHyperHeapCheck(pHeap);
615#endif
616
617 /*
618 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
619 */
620 if (pHeap->offFreeHead == NIL_OFFSET)
621 return NULL;
622
623 /*
624 * Page aligned chunks.
625 *
626 * Page aligned chunks can only be allocated from the last FREE chunk.
627 * This is for reasons of simplicity and fragmentation. Page aligned memory
628 * must also be allocated in page aligned sizes. Page aligned memory cannot
629 * be freed either.
630 *
631 * So, for this to work, the last FREE chunk needs to end on a page aligned
632 * boundary.
633 */
634 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
635 ASSERT_CHUNK_FREE(pHeap, pFree);
636 if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
637 || pFree->cb + sizeof(MMHYPERCHUNK) < cb)
638 {
639 Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
640 return NULL;
641 }
642
643 void *pvRet;
644 if (pFree->cb > cb)
645 {
646 /*
647 * Simple, just cut the top of the free node and return it.
648 */
649 pFree->cb -= cb;
650 pvRet = (char *)(&pFree->core + 1) + pFree->cb;
651 AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
652 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
653 pHeap->cbFree -= cb;
654 ASSERT_CHUNK_FREE(pHeap, pFree);
655 Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
656 }
657 else
658 {
659 /*
660 * Unlink the FREE node.
661 */
662 pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
663 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
664 pHeap->cbFree -= pFree->cb;
665
666 /* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
667 if (pvRet != (void *)pFree)
668 {
669 AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the *beep* did someone manage to allocated up all the heap with page aligned memory?!?\n"));
670 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
671 pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
672 AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
673#ifdef VBOX_WITH_STATISTICS
674 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
675 pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
676 pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
677#endif
678 Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
679 }
680
681 /* unlink from FREE chain. */
682 if (pFree->offPrev)
683 {
684 pHeap->offFreeTail += pFree->offPrev;
685 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
686 }
687 else
688 {
689 pHeap->offFreeTail = NIL_OFFSET;
690 pHeap->offFreeHead = NIL_OFFSET;
691 }
692 Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
693 }
694 pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
695 Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
696
697#ifdef MMHYPER_HEAP_STRICT
698 mmHyperHeapCheck(pHeap);
699#endif
700 return pvRet;
701}
702
703#ifdef VBOX_WITH_STATISTICS
704
705/**
706 * Get the statistic record for a tag.
707 *
708 * @returns Pointer to a stat record.
709 * @returns NULL on failure.
710 * @param pHeap The heap.
711 * @param enmTag The tag.
712 */
713static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
714{
715 /* try look it up first. */
716 PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
717 if (!pStat)
718 {
719 /* try allocate a new one */
720 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
721 if (!pChunk)
722 return NULL;
723 pStat = (PMMHYPERSTAT)(pChunk + 1);
724 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
725
726 ASMMemZero32(pStat, sizeof(*pStat));
727 pStat->Core.Key = enmTag;
728 RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
729 }
730 if (!pStat->fRegistered)
731 {
732# ifdef IN_RING3
733 mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
734# else
735 /** @todo schedule a R3 action. */
736# endif
737 }
738 return pStat;
739}
740
741
742# ifdef IN_RING3
743/**
744 * Registers statistics with STAM.
745 *
746 */
747static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
748{
749 if (pStat->fRegistered)
750 return;
751 const char *pszTag = mmGetTagName((MMTAG)pStat->Core.Key);
752 STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/HyperHeap/%s", pszTag);
753 STAMR3RegisterF(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.", "/MM/HyperHeap/%s/cAllocations", pszTag);
754 STAMR3RegisterF(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.", "/MM/HyperHeap/%s/cFrees", pszTag);
755 STAMR3RegisterF(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/HyperHeap/%s/cFailures", pszTag);
756 STAMR3RegisterF(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.", "/MM/HyperHeap/%s/cbAllocated", pszTag);
757 STAMR3RegisterF(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.", "/MM/HyperHeap/%s/cbFreed", pszTag);
758 STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
759 pStat->fRegistered = true;
760}
761# endif /* IN_RING3 */
762
763#endif /* VBOX_WITH_STATISTICS */
764
765
766/**
767 * Free memory allocated using MMHyperAlloc().
768 * The caller validates the parameters of this request.
769 *
770 * @returns VBox status code.
771 * @param pVM Pointer to the VM.
772 * @param pv The memory to free.
773 * @remark Try avoid free hyper memory.
774 */
775static int mmHyperFreeInternal(PVM pVM, void *pv)
776{
777 Log2(("MMHyperFree: pv=%p\n", pv));
778 if (!pv)
779 return VINF_SUCCESS;
780 AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
781 ("Invalid pointer %p!\n", pv),
782 VERR_INVALID_POINTER);
783
784 /*
785 * Get the heap and stats.
786 * Validate the chunk at the same time.
787 */
788 PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
789
790 AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
791 || RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
792 ("%p: offNext=%#RX32\n", pv, pChunk->offNext),
793 VERR_INVALID_POINTER);
794
795 AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
796 ("%p: Not used!\n", pv),
797 VERR_INVALID_POINTER);
798
799 int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
800 AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
801 && !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
802 ("%p: offPrev=%#RX32!\n", pv, offPrev),
803 VERR_INVALID_POINTER);
804
805 /* statistics */
806#ifdef VBOX_WITH_STATISTICS
807 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
808 AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
809 && pChunk->offStat,
810 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
811 VERR_INVALID_POINTER);
812#else
813 AssertMsgReturn(!pChunk->offStat,
814 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
815 VERR_INVALID_POINTER);
816#endif
817
818 /* The heap structure. */
819 PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
820 AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
821 && pChunk->offHeap,
822 ("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
823 VERR_INVALID_POINTER);
824
825 AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
826 ("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
827 VERR_INVALID_POINTER);
828 Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap));
829
830 /* Some more verifications using additional info from pHeap. */
831 AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
832 ("%p: offPrev=%#RX32!\n", pv, offPrev),
833 VERR_INVALID_POINTER);
834
835 AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
836 ("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
837 VERR_INVALID_POINTER);
838
839 AssertMsgReturn( (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
840 ("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
841 (char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
842 VERR_INVALID_POINTER);
843
844#ifdef MMHYPER_HEAP_STRICT
845 mmHyperHeapCheck(pHeap);
846#endif
847
848#if defined(VBOX_WITH_STATISTICS) || defined(MMHYPER_HEAP_FREE_POISON)
849 /* calc block size. */
850 const uint32_t cbChunk = pChunk->offNext
851 ? pChunk->offNext
852 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
853#endif
854#ifdef MMHYPER_HEAP_FREE_POISON
855 /* poison the block */
856 memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
857#endif
858
859#ifdef MMHYPER_HEAP_FREE_DELAY
860# ifdef MMHYPER_HEAP_FREE_POISON
861 /*
862 * Check poison.
863 */
864 unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
865 while (i-- > 0)
866 if (pHeap->aDelayedFrees[i].offChunk)
867 {
868 PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
869 const size_t cb = pCur->offNext
870 ? pCur->offNext - sizeof(*pCur)
871 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pCur - sizeof(*pCur);
872 uint8_t *pab = (uint8_t *)(pCur + 1);
873 for (unsigned off = 0; off < cb; off++)
874 AssertReleaseMsg(pab[off] == 0xCB,
875 ("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
876 pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
877 }
878# endif /* MMHYPER_HEAP_FREE_POISON */
879
880 /*
881 * Delayed freeing.
882 */
883 int rc = VINF_SUCCESS;
884 if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
885 {
886 PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
887 rc = mmHyperFree(pHeap, pChunkFree);
888 }
889 pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
890 pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
891 pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
892
893#else /* !MMHYPER_HEAP_FREE_POISON */
894 /*
895 * Call the worker.
896 */
897 int rc = mmHyperFree(pHeap, pChunk);
898#endif /* !MMHYPER_HEAP_FREE_POISON */
899
900 /*
901 * Update statistics.
902 */
903#ifdef VBOX_WITH_STATISTICS
904 pStat->cFrees++;
905 if (RT_SUCCESS(rc))
906 {
907 pStat->cbFreed += cbChunk;
908 pStat->cbCurAllocated -= cbChunk;
909 }
910 else
911 pStat->cFailures++;
912#endif
913
914 return rc;
915}
916
917
918/**
919 * Wrapper for mmHyperFreeInternal
920 */
921VMMDECL(int) MMHyperFree(PVM pVM, void *pv)
922{
923 int rc;
924
925 rc = mmHyperLock(pVM);
926 AssertRCReturn(rc, rc);
927
928 LogFlow(("MMHyperFree %p\n", pv));
929
930 rc = mmHyperFreeInternal(pVM, pv);
931
932 mmHyperUnlock(pVM);
933 return rc;
934}
935
936
937/**
938 * Free memory a memory chunk.
939 *
940 * @returns VBox status code.
941 * @param pHeap The heap.
942 * @param pChunk The memory chunk to free.
943 */
944static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
945{
946 Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
947 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
948
949 /*
950 * Insert into the free list (which is sorted on address).
951 *
952 * We'll search towards the end of the heap to locate the
953 * closest FREE chunk.
954 */
955 PMMHYPERCHUNKFREE pLeft = NULL;
956 PMMHYPERCHUNKFREE pRight = NULL;
957 if (pHeap->offFreeTail != NIL_OFFSET)
958 {
959 if (pFree->core.offNext)
960 {
961 pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
962 ASSERT_CHUNK(pHeap, &pRight->core);
963 while (!MMHYPERCHUNK_ISFREE(&pRight->core))
964 {
965 if (!pRight->core.offNext)
966 {
967 pRight = NULL;
968 break;
969 }
970 pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
971 ASSERT_CHUNK(pHeap, &pRight->core);
972 }
973 }
974 if (!pRight)
975 pRight = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /** @todo this can't be correct! 'pLeft = .. ; else' I think */
976 if (pRight)
977 {
978 ASSERT_CHUNK_FREE(pHeap, pRight);
979 if (pRight->offPrev)
980 {
981 pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
982 ASSERT_CHUNK_FREE(pHeap, pLeft);
983 }
984 }
985 }
986 if (pLeft == pFree)
987 {
988 AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
989 return VERR_INVALID_POINTER;
990 }
991 pChunk->offStat = 0;
992
993 /*
994 * Head free chunk list?
995 */
996 if (!pLeft)
997 {
998 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
999 pFree->offPrev = 0;
1000 pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1001 if (pRight)
1002 {
1003 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1004 pRight->offPrev = -(int32_t)pFree->offNext;
1005 }
1006 else
1007 {
1008 pFree->offNext = 0;
1009 pHeap->offFreeTail = pHeap->offFreeHead;
1010 }
1011 Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
1012 }
1013 else
1014 {
1015 /*
1016 * Can we merge with left hand free chunk?
1017 */
1018 if ((char *)pLeft + pLeft->core.offNext == (char *)pFree)
1019 {
1020 if (pFree->core.offNext)
1021 {
1022 pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
1023 MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
1024 }
1025 else
1026 pLeft->core.offNext = 0;
1027 pFree = pLeft;
1028 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
1029 pHeap->cbFree -= pLeft->cb;
1030 Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
1031 }
1032 /*
1033 * No, just link it into the free list then.
1034 */
1035 else
1036 {
1037 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
1038 pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
1039 pLeft->offNext = -pFree->offPrev;
1040 if (pRight)
1041 {
1042 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1043 pRight->offPrev = -(int32_t)pFree->offNext;
1044 }
1045 else
1046 {
1047 pFree->offNext = 0;
1048 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1049 }
1050 Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
1051 }
1052 }
1053
1054 /*
1055 * Can we merge with right hand free chunk?
1056 */
1057 if (pRight && (char *)pRight == (char *)pFree + pFree->core.offNext)
1058 {
1059 /* core */
1060 if (pRight->core.offNext)
1061 {
1062 pFree->core.offNext += pRight->core.offNext;
1063 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
1064 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
1065 ASSERT_CHUNK(pHeap, pNext);
1066 }
1067 else
1068 pFree->core.offNext = 0;
1069
1070 /* free */
1071 if (pRight->offNext)
1072 {
1073 pFree->offNext += pRight->offNext;
1074 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
1075 }
1076 else
1077 {
1078 pFree->offNext = 0;
1079 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1080 }
1081 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1082 pHeap->cbFree -= pRight->cb;
1083 Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1084 }
1085
1086 /* calculate the size. */
1087 if (pFree->core.offNext)
1088 pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1089 else
1090 pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
1091 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1092 pHeap->cbFree += pFree->cb;
1093 ASSERT_CHUNK_FREE(pHeap, pFree);
1094
1095#ifdef MMHYPER_HEAP_STRICT
1096 mmHyperHeapCheck(pHeap);
1097#endif
1098 return VINF_SUCCESS;
1099}
1100
1101
1102#if defined(DEBUG) || defined(MMHYPER_HEAP_STRICT)
1103/**
1104 * Dumps a heap chunk to the log.
1105 *
1106 * @param pHeap Pointer to the heap.
1107 * @param pCur Pointer to the chunk.
1108 */
1109static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1110{
1111 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1112 {
1113 if (pCur->core.offStat)
1114 {
1115 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1116 const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1117#ifdef IN_RING3
1118 Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1119 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1120 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1121 mmGetTagName((MMTAG)pStat->Core.Key), pszSelf));
1122#else
1123 Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1124 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1125 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1126 (MMTAG)pStat->Core.Key, pszSelf));
1127#endif
1128 }
1129 else
1130 Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1131 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1132 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1133 }
1134 else
1135 Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1136 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1137 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1138}
1139#endif /* DEBUG || MMHYPER_HEAP_STRICT */
1140
1141
1142#ifdef MMHYPER_HEAP_STRICT
1143/**
1144 * Internal consistency check.
1145 */
1146static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1147{
1148 PMMHYPERCHUNKFREE pPrev = NULL;
1149 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1150 for (;;)
1151 {
1152 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1153 ASSERT_CHUNK_USED(pHeap, &pCur->core);
1154 else
1155 ASSERT_CHUNK_FREE(pHeap, pCur);
1156 if (pPrev)
1157 AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1158 ("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1159
1160# ifdef MMHYPER_HEAP_STRICT_FENCE
1161 uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
1162 if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1163 && off < pHeap->offPageAligned)
1164 {
1165 uint32_t cbCur = pCur->core.offNext
1166 ? pCur->core.offNext
1167 : pHeap->cbHeap - off;
1168 uint32_t *pu32End = ((uint32_t *)((uint8_t *)pCur + cbCur));
1169 uint32_t cbFence = pu32End[-1];
1170 if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1171 || cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1172 {
1173 mmHyperHeapDumpOne(pHeap, pCur);
1174 Assert(cbFence < cbCur - sizeof(*pCur));
1175 Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1176 }
1177
1178 uint32_t *pu32Bad = ASMMemIsAllU32((uint8_t *)pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1179 if (RT_UNLIKELY(pu32Bad))
1180 {
1181 mmHyperHeapDumpOne(pHeap, pCur);
1182 Assert(!pu32Bad);
1183 }
1184 }
1185# endif
1186
1187 /* next */
1188 if (!pCur->core.offNext)
1189 break;
1190 pPrev = pCur;
1191 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1192 }
1193}
1194#endif
1195
1196
1197/**
1198 * Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1199 * defined at build time.
1200 *
1201 * @param pVM Pointer to the VM.
1202 */
1203VMMDECL(void) MMHyperHeapCheck(PVM pVM)
1204{
1205#ifdef MMHYPER_HEAP_STRICT
1206 int rc;
1207
1208 rc = mmHyperLock(pVM);
1209 AssertRC(rc);
1210 mmHyperHeapCheck(pVM->mm.s.CTX_SUFF(pHyperHeap));
1211 mmHyperUnlock(pVM);
1212#endif
1213}
1214
1215
1216#ifdef DEBUG
1217/**
1218 * Dumps the hypervisor heap to Log.
1219 * @param pVM Pointer to the VM.
1220 */
1221VMMDECL(void) MMHyperHeapDump(PVM pVM)
1222{
1223 Log(("MMHyperHeapDump: *** heap dump - start ***\n"));
1224 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
1225 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1226 for (;;)
1227 {
1228 mmHyperHeapDumpOne(pHeap, pCur);
1229
1230 /* next */
1231 if (!pCur->core.offNext)
1232 break;
1233 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1234 }
1235 Log(("MMHyperHeapDump: *** heap dump - end ***\n"));
1236}
1237#endif
1238
1239
1240/**
1241 * Query the amount of free memory in the hypervisor heap.
1242 *
1243 * @returns Number of free bytes in the hypervisor heap.
1244 */
1245VMMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1246{
1247 return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbFree;
1248}
1249
1250/**
1251 * Query the size the hypervisor heap.
1252 *
1253 * @returns The size of the hypervisor heap in bytes.
1254 */
1255VMMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1256{
1257 return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap;
1258}
1259
1260
1261/**
1262 * Query the address and size the hypervisor memory area.
1263 *
1264 * @returns Base address of the hypervisor area.
1265 * @param pVM Pointer to the VM.
1266 * @param pcb Where to store the size of the hypervisor area. (out)
1267 */
1268VMMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1269{
1270 if (pcb)
1271 *pcb = pVM->mm.s.cbHyperArea;
1272 return pVM->mm.s.pvHyperAreaGC;
1273}
1274
1275
1276/**
1277 * Checks if an address is within the hypervisor memory area.
1278 *
1279 * @returns true if inside.
1280 * @returns false if outside.
1281 * @param pVM Pointer to the VM.
1282 * @param GCPtr The pointer to check.
1283 */
1284VMMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1285{
1286 return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1287}
1288
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use