VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 43667

Last change on this file since 43667 was 41965, checked in by vboxsync, 12 years ago

VMM: ran scm. Mostly svn:keywords changes (adding Revision).
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 47.1 KB
Line 
1/* $Id: MMAllHyper.cpp 41965 2012-06-29 02:52:49Z vboxsync $ */
2/** @file
3 * MM - Memory Manager - Hypervisor Memory Area, All Contexts.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
23#include <VBox/vmm/mm.h>
24#include <VBox/vmm/stam.h>
25#include "MMInternal.h"
26#include <VBox/vmm/vm.h>
27
28#include <VBox/err.h>
29#include <VBox/param.h>
30#include <iprt/assert.h>
31#include <VBox/log.h>
32#include <iprt/asm.h>
33#include <iprt/string.h>
34
35
36/*******************************************************************************
37* Defined Constants And Macros *
38*******************************************************************************/
39#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
40#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
41#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
42#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
43
44#define ASSERT_OFFPREV(pHeap, pChunk) \
45 do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
46 Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
47 AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
48 || (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
49 ("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
50 } while (0)
51
52#define ASSERT_OFFNEXT(pHeap, pChunk) \
53 do { ASSERT_ALIGN((pChunk)->offNext); \
54 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
55 } while (0)
56
57#define ASSERT_OFFHEAP(pHeap, pChunk) \
58 do { Assert((pChunk)->offHeap); \
59 AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
60 ("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
61 Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
62 } while (0)
63
64#ifdef VBOX_WITH_STATISTICS
65#define ASSERT_OFFSTAT(pHeap, pChunk) \
66 do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
67 Assert(!(pChunk)->offStat); \
68 else if ((pChunk)->offStat) \
69 { \
70 Assert((pChunk)->offStat); \
71 AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
72 uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
73 AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
74 ("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
75 } \
76 } while (0)
77#else
78#define ASSERT_OFFSTAT(pHeap, pChunk) \
79 do { Assert(!(pChunk)->offStat); \
80 } while (0)
81#endif
82
83#define ASSERT_CHUNK(pHeap, pChunk) \
84 do { ASSERT_OFFNEXT(pHeap, pChunk); \
85 ASSERT_OFFPREV(pHeap, pChunk); \
86 ASSERT_OFFHEAP(pHeap, pChunk); \
87 ASSERT_OFFSTAT(pHeap, pChunk); \
88 } while (0)
89#define ASSERT_CHUNK_USED(pHeap, pChunk) \
90 do { ASSERT_OFFNEXT(pHeap, pChunk); \
91 ASSERT_OFFPREV(pHeap, pChunk); \
92 Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
93 } while (0)
94
95#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
96 do { ASSERT_ALIGN((pChunk)->offPrev); \
97 ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
98 Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) || !(pChunk)->offPrev); \
99 AssertMsg( (pChunk)->offPrev \
100 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
101 ("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
102 (pHeap)->offFreeHead)); \
103 } while (0)
104
105#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
106 do { ASSERT_ALIGN((pChunk)->offNext); \
107 ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
108 Assert((pChunk)->offNext != (pChunk)->core.offNext || !(pChunk)->offNext); \
109 AssertMsg( (pChunk)->offNext \
110 || (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
111 ("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
112 (pHeap)->offFreeTail)); \
113 } while (0)
114
115#define ASSERT_FREE_CB(pHeap, pChunk) \
116 do { ASSERT_ALIGN((pChunk)->cb); \
117 Assert((pChunk)->cb > 0); \
118 if ((pChunk)->core.offNext) \
119 AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
120 ("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
121 else \
122 ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
123 } while (0)
124
125#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
126 do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
127 Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
128 ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
129 ASSERT_FREE_OFFPREV(pHeap, pChunk); \
130 ASSERT_FREE_CB(pHeap, pChunk); \
131 } while (0)
132
133
134/*******************************************************************************
135* Internal Functions *
136*******************************************************************************/
137static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
138static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
139#ifdef VBOX_WITH_STATISTICS
140static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
141#ifdef IN_RING3
142static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
143#endif
144#endif
145static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
146#ifdef MMHYPER_HEAP_STRICT
147static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
148#endif
149
150
151
152/**
153 * Locks the hypervisor heap.
154 * This might call back to Ring-3 in order to deal with lock contention in GC and R3.
155 *
156 * @param pVM Pointer to the VM.
157 */
158static int mmHyperLock(PVM pVM)
159{
160 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
161
162#ifdef IN_RING3
163 if (!PDMCritSectIsInitialized(&pHeap->Lock))
164 return VINF_SUCCESS; /* early init */
165#else
166 Assert(PDMCritSectIsInitialized(&pHeap->Lock));
167#endif
168 int rc = PDMCritSectEnter(&pHeap->Lock, VERR_SEM_BUSY);
169#if defined(IN_RC) || defined(IN_RING0)
170 if (rc == VERR_SEM_BUSY)
171 rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_MMHYPER_LOCK, 0);
172#endif
173 AssertRC(rc);
174 return rc;
175}
176
177
178/**
179 * Unlocks the hypervisor heap.
180 *
181 * @param pVM Pointer to the VM.
182 */
183static void mmHyperUnlock(PVM pVM)
184{
185 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
186
187#ifdef IN_RING3
188 if (!PDMCritSectIsInitialized(&pHeap->Lock))
189 return; /* early init */
190#endif
191 Assert(PDMCritSectIsInitialized(&pHeap->Lock));
192 PDMCritSectLeave(&pHeap->Lock);
193}
194
195/**
196 * Allocates memory in the Hypervisor (RC VMM) area.
197 * The returned memory is of course zeroed.
198 *
199 * @returns VBox status code.
200 * @param pVM Pointer to the VM.
201 * @param cb Number of bytes to allocate.
202 * @param uAlignment Required memory alignment in bytes.
203 * Values are 0,8,16,32,64 and PAGE_SIZE.
204 * 0 -> default alignment, i.e. 8 bytes.
205 * @param enmTag The statistics tag.
206 * @param ppv Where to store the address to the allocated
207 * memory.
208 */
209static int mmHyperAllocInternal(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
210{
211 AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
212
213 /*
214 * Validate input and adjust it to reasonable values.
215 */
216 if (!uAlignment || uAlignment < MMHYPER_HEAP_ALIGN_MIN)
217 uAlignment = MMHYPER_HEAP_ALIGN_MIN;
218 uint32_t cbAligned;
219 switch (uAlignment)
220 {
221 case 8:
222 case 16:
223 case 32:
224 case 64:
225 cbAligned = RT_ALIGN_32(cb, MMHYPER_HEAP_ALIGN_MIN);
226 if (!cbAligned || cbAligned < cb)
227 {
228 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
229 AssertMsgFailed(("Nice try.\n"));
230 return VERR_INVALID_PARAMETER;
231 }
232 break;
233
234 case PAGE_SIZE:
235 AssertMsg(RT_ALIGN_32(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
236 cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
237 if (!cbAligned)
238 {
239 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
240 AssertMsgFailed(("Nice try.\n"));
241 return VERR_INVALID_PARAMETER;
242 }
243 break;
244
245 default:
246 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
247 AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
248 return VERR_INVALID_PARAMETER;
249 }
250
251
252 /*
253 * Get heap and statisticsStatistics.
254 */
255 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
256#ifdef VBOX_WITH_STATISTICS
257 PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
258 if (!pStat)
259 {
260 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
261 AssertMsgFailed(("Failed to allocate statistics!\n"));
262 return VERR_MM_HYPER_NO_MEMORY;
263 }
264#endif
265 if (uAlignment < PAGE_SIZE)
266 {
267 /*
268 * Allocate a chunk.
269 */
270 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
271 if (pChunk)
272 {
273#ifdef VBOX_WITH_STATISTICS
274 const uint32_t cbChunk = pChunk->offNext
275 ? pChunk->offNext
276 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
277 pStat->cbAllocated += (uint32_t)cbChunk;
278 pStat->cbCurAllocated += (uint32_t)cbChunk;
279 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
280 pStat->cbMaxAllocated = pStat->cbCurAllocated;
281 pStat->cAllocations++;
282 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
283#else
284 pChunk->offStat = 0;
285#endif
286 void *pv = pChunk + 1;
287 *ppv = pv;
288 ASMMemZero32(pv, cbAligned);
289 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
290 return VINF_SUCCESS;
291 }
292 }
293 else
294 {
295 /*
296 * Allocate page aligned memory.
297 */
298 void *pv = mmHyperAllocPages(pHeap, cbAligned);
299 if (pv)
300 {
301#ifdef VBOX_WITH_STATISTICS
302 pStat->cbAllocated += cbAligned;
303 pStat->cbCurAllocated += cbAligned;
304 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
305 pStat->cbMaxAllocated = pStat->cbCurAllocated;
306 pStat->cAllocations++;
307#endif
308 *ppv = pv;
309 /* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPR3PageAlloc zeros it. */
310 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
311 return VINF_SUCCESS;
312 }
313 }
314
315#ifdef VBOX_WITH_STATISTICS
316 pStat->cAllocations++;
317 pStat->cFailures++;
318#endif
319 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
320 AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
321 return VERR_MM_HYPER_NO_MEMORY;
322}
323
324/**
325 * Wrapper for mmHyperAllocInternal
326 */
327VMMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
328{
329 int rc;
330
331 rc = mmHyperLock(pVM);
332 AssertRCReturn(rc, rc);
333
334 LogFlow(("MMHyperAlloc %x align=%x tag=%s\n", cb, uAlignment, mmGetTagName(enmTag)));
335
336 rc = mmHyperAllocInternal(pVM, cb, uAlignment, enmTag, ppv);
337
338 mmHyperUnlock(pVM);
339 return rc;
340}
341
342/**
343 * Allocates a chunk of memory from the specified heap.
344 * The caller validates the parameters of this request.
345 *
346 * @returns Pointer to the allocated chunk.
347 * @returns NULL on failure.
348 * @param pHeap The heap.
349 * @param cb Size of the memory block to allocate.
350 * @param uAlignment The alignment specifications for the allocated block.
351 * @internal
352 */
353static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
354{
355 Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
356#ifdef MMHYPER_HEAP_STRICT
357 mmHyperHeapCheck(pHeap);
358#endif
359#ifdef MMHYPER_HEAP_STRICT_FENCE
360 uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
361 cb += cbFence;
362#endif
363
364 /*
365 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
366 */
367 if (pHeap->offFreeHead == NIL_OFFSET)
368 return NULL;
369
370 /*
371 * Small alignments - from the front of the heap.
372 *
373 * Must split off free chunks at the end to prevent messing up the
374 * last free node which we take the page aligned memory from the top of.
375 */
376 PMMHYPERCHUNK pRet = NULL;
377 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
378 while (pFree)
379 {
380 ASSERT_CHUNK_FREE(pHeap, pFree);
381 if (pFree->cb >= cb)
382 {
383 unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
384 if (offAlign)
385 offAlign = uAlignment - offAlign;
386 if (!offAlign || pFree->cb - offAlign >= cb)
387 {
388 Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
389
390 /*
391 * Adjust the node in front.
392 * Because of multiple alignments we need to special case allocation of the first block.
393 */
394 if (offAlign)
395 {
396 MMHYPERCHUNKFREE Free = *pFree;
397 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
398 {
399 /* just add a bit of memory to it. */
400 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
401 pPrev->core.offNext += offAlign;
402 AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
403 Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
404 }
405 else
406 {
407 /* make new head node, mark it USED for simplicity. */
408 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
409 Assert(pPrev == &pFree->core);
410 pPrev->offPrev = 0;
411 MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
412 pPrev->offNext = offAlign;
413 Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
414
415 }
416 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
417 pHeap->cbFree -= offAlign;
418
419 /* Recreate pFree node and adjusting everything... */
420 pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
421 *pFree = Free;
422
423 pFree->cb -= offAlign;
424 if (pFree->core.offNext)
425 {
426 pFree->core.offNext -= offAlign;
427 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
428 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
429 ASSERT_CHUNK(pHeap, pNext);
430 }
431 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
432 MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
433
434 if (pFree->offNext)
435 {
436 pFree->offNext -= offAlign;
437 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
438 pNext->offPrev = -(int32_t)pFree->offNext;
439 ASSERT_CHUNK_FREE(pHeap, pNext);
440 }
441 else
442 pHeap->offFreeTail += offAlign;
443 if (pFree->offPrev)
444 {
445 pFree->offPrev -= offAlign;
446 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
447 pPrev->offNext = -pFree->offPrev;
448 ASSERT_CHUNK_FREE(pHeap, pPrev);
449 }
450 else
451 pHeap->offFreeHead += offAlign;
452 pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
453 pFree->core.offStat = 0;
454 ASSERT_CHUNK_FREE(pHeap, pFree);
455 Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
456 }
457
458 /*
459 * Split off a new FREE chunk?
460 */
461 if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
462 {
463 /*
464 * Move the FREE chunk up to make room for the new USED chunk.
465 */
466 const int off = cb + sizeof(MMHYPERCHUNK);
467 PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
468 *pNew = *pFree;
469 pNew->cb -= off;
470 if (pNew->core.offNext)
471 {
472 pNew->core.offNext -= off;
473 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
474 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
475 ASSERT_CHUNK(pHeap, pNext);
476 }
477 pNew->core.offPrev = -off;
478 MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
479
480 if (pNew->offNext)
481 {
482 pNew->offNext -= off;
483 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
484 pNext->offPrev = -(int32_t)pNew->offNext;
485 ASSERT_CHUNK_FREE(pHeap, pNext);
486 }
487 else
488 pHeap->offFreeTail += off;
489 if (pNew->offPrev)
490 {
491 pNew->offPrev -= off;
492 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
493 pPrev->offNext = -pNew->offPrev;
494 ASSERT_CHUNK_FREE(pHeap, pPrev);
495 }
496 else
497 pHeap->offFreeHead += off;
498 pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
499 pNew->core.offStat = 0;
500 ASSERT_CHUNK_FREE(pHeap, pNew);
501
502 /*
503 * Update the old FREE node making it a USED node.
504 */
505 pFree->core.offNext = off;
506 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
507
508
509 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
510 pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
511 pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
512 pRet = &pFree->core;
513 ASSERT_CHUNK(pHeap, &pFree->core);
514 Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
515 }
516 else
517 {
518 /*
519 * Link out of free list.
520 */
521 if (pFree->offNext)
522 {
523 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
524 if (pFree->offPrev)
525 {
526 pNext->offPrev += pFree->offPrev;
527 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
528 pPrev->offNext += pFree->offNext;
529 ASSERT_CHUNK_FREE(pHeap, pPrev);
530 }
531 else
532 {
533 pHeap->offFreeHead += pFree->offNext;
534 pNext->offPrev = 0;
535 }
536 ASSERT_CHUNK_FREE(pHeap, pNext);
537 }
538 else
539 {
540 if (pFree->offPrev)
541 {
542 pHeap->offFreeTail += pFree->offPrev;
543 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
544 pPrev->offNext = 0;
545 ASSERT_CHUNK_FREE(pHeap, pPrev);
546 }
547 else
548 {
549 pHeap->offFreeHead = NIL_OFFSET;
550 pHeap->offFreeTail = NIL_OFFSET;
551 }
552 }
553
554 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
555 pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
556 pHeap->cbFree -= pFree->cb;
557 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
558 pRet = &pFree->core;
559 ASSERT_CHUNK(pHeap, &pFree->core);
560 Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
561 }
562 Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
563 break;
564 }
565 }
566
567 /* next */
568 pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
569 }
570
571#ifdef MMHYPER_HEAP_STRICT_FENCE
572 uint32_t *pu32End = (uint32_t *)((uint8_t *)(pRet + 1) + cb);
573 uint32_t *pu32EndReal = pRet->offNext
574 ? (uint32_t *)((uint8_t *)pRet + pRet->offNext)
575 : (uint32_t *)(pHeap->CTX_SUFF(pbHeap) + pHeap->cbHeap);
576 cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
577 ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
578 pu32EndReal[-1] = cbFence;
579#endif
580#ifdef MMHYPER_HEAP_STRICT
581 mmHyperHeapCheck(pHeap);
582#endif
583 return pRet;
584}
585
586
587/**
588 * Allocates one or more pages of memory from the specified heap.
589 * The caller validates the parameters of this request.
590 *
591 * @returns Pointer to the allocated chunk.
592 * @returns NULL on failure.
593 * @param pHeap The heap.
594 * @param cb Size of the memory block to allocate.
595 * @internal
596 */
597static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
598{
599 Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
600
601#ifdef MMHYPER_HEAP_STRICT
602 mmHyperHeapCheck(pHeap);
603#endif
604
605 /*
606 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
607 */
608 if (pHeap->offFreeHead == NIL_OFFSET)
609 return NULL;
610
611 /*
612 * Page aligned chunks.
613 *
614 * Page aligned chunks can only be allocated from the last FREE chunk.
615 * This is for reasons of simplicity and fragmentation. Page aligned memory
616 * must also be allocated in page aligned sizes. Page aligned memory cannot
617 * be freed either.
618 *
619 * So, for this to work, the last FREE chunk needs to end on a page aligned
620 * boundary.
621 */
622 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
623 ASSERT_CHUNK_FREE(pHeap, pFree);
624 if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
625 || pFree->cb + sizeof(MMHYPERCHUNK) < cb)
626 {
627 Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
628 return NULL;
629 }
630
631 void *pvRet;
632 if (pFree->cb > cb)
633 {
634 /*
635 * Simple, just cut the top of the free node and return it.
636 */
637 pFree->cb -= cb;
638 pvRet = (char *)(&pFree->core + 1) + pFree->cb;
639 AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
640 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
641 pHeap->cbFree -= cb;
642 ASSERT_CHUNK_FREE(pHeap, pFree);
643 Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
644 }
645 else
646 {
647 /*
648 * Unlink the FREE node.
649 */
650 pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
651 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
652 pHeap->cbFree -= pFree->cb;
653
654 /* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
655 if (pvRet != (void *)pFree)
656 {
657 AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the *beep* did someone manage to allocated up all the heap with page aligned memory?!?\n"));
658 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
659 pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
660 AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
661#ifdef VBOX_WITH_STATISTICS
662 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
663 pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
664 pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
665#endif
666 Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
667 }
668
669 /* unlink from FREE chain. */
670 if (pFree->offPrev)
671 {
672 pHeap->offFreeTail += pFree->offPrev;
673 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
674 }
675 else
676 {
677 pHeap->offFreeTail = NIL_OFFSET;
678 pHeap->offFreeHead = NIL_OFFSET;
679 }
680 Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
681 }
682 pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
683 Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
684
685#ifdef MMHYPER_HEAP_STRICT
686 mmHyperHeapCheck(pHeap);
687#endif
688 return pvRet;
689}
690
691#ifdef VBOX_WITH_STATISTICS
692
693/**
694 * Get the statistic record for a tag.
695 *
696 * @returns Pointer to a stat record.
697 * @returns NULL on failure.
698 * @param pHeap The heap.
699 * @param enmTag The tag.
700 */
701static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
702{
703 /* try look it up first. */
704 PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
705 if (!pStat)
706 {
707 /* try allocate a new one */
708 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
709 if (!pChunk)
710 return NULL;
711 pStat = (PMMHYPERSTAT)(pChunk + 1);
712 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
713
714 ASMMemZero32(pStat, sizeof(*pStat));
715 pStat->Core.Key = enmTag;
716 RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
717 }
718 if (!pStat->fRegistered)
719 {
720# ifdef IN_RING3
721 mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
722# else
723 /** @todo schedule a R3 action. */
724# endif
725 }
726 return pStat;
727}
728
729
730# ifdef IN_RING3
731/**
732 * Registers statistics with STAM.
733 *
734 */
735static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
736{
737 if (pStat->fRegistered)
738 return;
739 const char *pszTag = mmGetTagName((MMTAG)pStat->Core.Key);
740 STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/HyperHeap/%s", pszTag);
741 STAMR3RegisterF(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.", "/MM/HyperHeap/%s/cAllocations", pszTag);
742 STAMR3RegisterF(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.", "/MM/HyperHeap/%s/cFrees", pszTag);
743 STAMR3RegisterF(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/HyperHeap/%s/cFailures", pszTag);
744 STAMR3RegisterF(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.", "/MM/HyperHeap/%s/cbAllocated", pszTag);
745 STAMR3RegisterF(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.", "/MM/HyperHeap/%s/cbFreed", pszTag);
746 STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
747 pStat->fRegistered = true;
748}
749# endif /* IN_RING3 */
750
751#endif /* VBOX_WITH_STATISTICS */
752
753
754/**
755 * Free memory allocated using MMHyperAlloc().
756 * The caller validates the parameters of this request.
757 *
758 * @returns VBox status code.
759 * @param pVM Pointer to the VM.
760 * @param pv The memory to free.
761 * @remark Try avoid free hyper memory.
762 */
763static int mmHyperFreeInternal(PVM pVM, void *pv)
764{
765 Log2(("MMHyperFree: pv=%p\n", pv));
766 if (!pv)
767 return VINF_SUCCESS;
768 AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
769 ("Invalid pointer %p!\n", pv),
770 VERR_INVALID_POINTER);
771
772 /*
773 * Get the heap and stats.
774 * Validate the chunk at the same time.
775 */
776 PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
777
778 AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
779 || RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
780 ("%p: offNext=%#RX32\n", pv, pChunk->offNext),
781 VERR_INVALID_POINTER);
782
783 AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
784 ("%p: Not used!\n", pv),
785 VERR_INVALID_POINTER);
786
787 int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
788 AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
789 && !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
790 ("%p: offPrev=%#RX32!\n", pv, offPrev),
791 VERR_INVALID_POINTER);
792
793 /* statistics */
794#ifdef VBOX_WITH_STATISTICS
795 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
796 AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
797 && pChunk->offStat,
798 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
799 VERR_INVALID_POINTER);
800#else
801 AssertMsgReturn(!pChunk->offStat,
802 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
803 VERR_INVALID_POINTER);
804#endif
805
806 /* The heap structure. */
807 PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
808 AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
809 && pChunk->offHeap,
810 ("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
811 VERR_INVALID_POINTER);
812
813 AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
814 ("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
815 VERR_INVALID_POINTER);
816 Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap));
817
818 /* Some more verifications using additional info from pHeap. */
819 AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
820 ("%p: offPrev=%#RX32!\n", pv, offPrev),
821 VERR_INVALID_POINTER);
822
823 AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
824 ("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
825 VERR_INVALID_POINTER);
826
827 AssertMsgReturn( (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
828 ("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
829 (char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
830 VERR_INVALID_POINTER);
831
832#ifdef MMHYPER_HEAP_STRICT
833 mmHyperHeapCheck(pHeap);
834#endif
835
836#if defined(VBOX_WITH_STATISTICS) || defined(MMHYPER_HEAP_FREE_POISON)
837 /* calc block size. */
838 const uint32_t cbChunk = pChunk->offNext
839 ? pChunk->offNext
840 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
841#endif
842#ifdef MMHYPER_HEAP_FREE_POISON
843 /* poison the block */
844 memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
845#endif
846
847#ifdef MMHYPER_HEAP_FREE_DELAY
848# ifdef MMHYPER_HEAP_FREE_POISON
849 /*
850 * Check poison.
851 */
852 unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
853 while (i-- > 0)
854 if (pHeap->aDelayedFrees[i].offChunk)
855 {
856 PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
857 const size_t cb = pCur->offNext
858 ? pCur->offNext - sizeof(*pCur)
859 : pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pCur - sizeof(*pCur);
860 uint8_t *pab = (uint8_t *)(pCur + 1);
861 for (unsigned off = 0; off < cb; off++)
862 AssertReleaseMsg(pab[off] == 0xCB,
863 ("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
864 pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
865 }
866# endif /* MMHYPER_HEAP_FREE_POISON */
867
868 /*
869 * Delayed freeing.
870 */
871 int rc = VINF_SUCCESS;
872 if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
873 {
874 PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
875 rc = mmHyperFree(pHeap, pChunkFree);
876 }
877 pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
878 pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
879 pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
880
881#else /* !MMHYPER_HEAP_FREE_POISON */
882 /*
883 * Call the worker.
884 */
885 int rc = mmHyperFree(pHeap, pChunk);
886#endif /* !MMHYPER_HEAP_FREE_POISON */
887
888 /*
889 * Update statistics.
890 */
891#ifdef VBOX_WITH_STATISTICS
892 pStat->cFrees++;
893 if (RT_SUCCESS(rc))
894 {
895 pStat->cbFreed += cbChunk;
896 pStat->cbCurAllocated -= cbChunk;
897 }
898 else
899 pStat->cFailures++;
900#endif
901
902 return rc;
903}
904
905
906/**
907 * Wrapper for mmHyperFreeInternal
908 */
909VMMDECL(int) MMHyperFree(PVM pVM, void *pv)
910{
911 int rc;
912
913 rc = mmHyperLock(pVM);
914 AssertRCReturn(rc, rc);
915
916 LogFlow(("MMHyperFree %p\n", pv));
917
918 rc = mmHyperFreeInternal(pVM, pv);
919
920 mmHyperUnlock(pVM);
921 return rc;
922}
923
924
925/**
926 * Free memory a memory chunk.
927 *
928 * @returns VBox status code.
929 * @param pHeap The heap.
930 * @param pChunk The memory chunk to free.
931 */
932static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
933{
934 Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
935 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
936
937 /*
938 * Insert into the free list (which is sorted on address).
939 *
940 * We'll search towards the end of the heap to locate the
941 * closest FREE chunk.
942 */
943 PMMHYPERCHUNKFREE pLeft = NULL;
944 PMMHYPERCHUNKFREE pRight = NULL;
945 if (pHeap->offFreeTail != NIL_OFFSET)
946 {
947 if (pFree->core.offNext)
948 {
949 pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
950 ASSERT_CHUNK(pHeap, &pRight->core);
951 while (!MMHYPERCHUNK_ISFREE(&pRight->core))
952 {
953 if (!pRight->core.offNext)
954 {
955 pRight = NULL;
956 break;
957 }
958 pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
959 ASSERT_CHUNK(pHeap, &pRight->core);
960 }
961 }
962 if (!pRight)
963 pRight = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /** @todo this can't be correct! 'pLeft = .. ; else' I think */
964 if (pRight)
965 {
966 ASSERT_CHUNK_FREE(pHeap, pRight);
967 if (pRight->offPrev)
968 {
969 pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
970 ASSERT_CHUNK_FREE(pHeap, pLeft);
971 }
972 }
973 }
974 if (pLeft == pFree)
975 {
976 AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
977 return VERR_INVALID_POINTER;
978 }
979 pChunk->offStat = 0;
980
981 /*
982 * Head free chunk list?
983 */
984 if (!pLeft)
985 {
986 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
987 pFree->offPrev = 0;
988 pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
989 if (pRight)
990 {
991 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
992 pRight->offPrev = -(int32_t)pFree->offNext;
993 }
994 else
995 {
996 pFree->offNext = 0;
997 pHeap->offFreeTail = pHeap->offFreeHead;
998 }
999 Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
1000 }
1001 else
1002 {
1003 /*
1004 * Can we merge with left hand free chunk?
1005 */
1006 if ((char *)pLeft + pLeft->core.offNext == (char *)pFree)
1007 {
1008 if (pFree->core.offNext)
1009 {
1010 pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
1011 MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
1012 }
1013 else
1014 pLeft->core.offNext = 0;
1015 pFree = pLeft;
1016 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
1017 pHeap->cbFree -= pLeft->cb;
1018 Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
1019 }
1020 /*
1021 * No, just link it into the free list then.
1022 */
1023 else
1024 {
1025 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
1026 pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
1027 pLeft->offNext = -pFree->offPrev;
1028 if (pRight)
1029 {
1030 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1031 pRight->offPrev = -(int32_t)pFree->offNext;
1032 }
1033 else
1034 {
1035 pFree->offNext = 0;
1036 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1037 }
1038 Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
1039 }
1040 }
1041
1042 /*
1043 * Can we merge with right hand free chunk?
1044 */
1045 if (pRight && (char *)pRight == (char *)pFree + pFree->core.offNext)
1046 {
1047 /* core */
1048 if (pRight->core.offNext)
1049 {
1050 pFree->core.offNext += pRight->core.offNext;
1051 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
1052 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
1053 ASSERT_CHUNK(pHeap, pNext);
1054 }
1055 else
1056 pFree->core.offNext = 0;
1057
1058 /* free */
1059 if (pRight->offNext)
1060 {
1061 pFree->offNext += pRight->offNext;
1062 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
1063 }
1064 else
1065 {
1066 pFree->offNext = 0;
1067 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1068 }
1069 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1070 pHeap->cbFree -= pRight->cb;
1071 Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1072 }
1073
1074 /* calculate the size. */
1075 if (pFree->core.offNext)
1076 pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1077 else
1078 pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
1079 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1080 pHeap->cbFree += pFree->cb;
1081 ASSERT_CHUNK_FREE(pHeap, pFree);
1082
1083#ifdef MMHYPER_HEAP_STRICT
1084 mmHyperHeapCheck(pHeap);
1085#endif
1086 return VINF_SUCCESS;
1087}
1088
1089
1090#if defined(DEBUG) || defined(MMHYPER_HEAP_STRICT)
1091/**
1092 * Dumps a heap chunk to the log.
1093 *
1094 * @param pHeap Pointer to the heap.
1095 * @param pCur Pointer to the chunk.
1096 */
1097static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1098{
1099 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1100 {
1101 if (pCur->core.offStat)
1102 {
1103 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1104 const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1105#ifdef IN_RING3
1106 Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1107 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1108 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1109 mmGetTagName((MMTAG)pStat->Core.Key), pszSelf));
1110#else
1111 Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1112 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1113 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1114 (MMTAG)pStat->Core.Key, pszSelf));
1115#endif
1116 }
1117 else
1118 Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1119 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1120 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1121 }
1122 else
1123 Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1124 pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1125 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1126}
1127#endif /* DEBUG || MMHYPER_HEAP_STRICT */
1128
1129
1130#ifdef MMHYPER_HEAP_STRICT
1131/**
1132 * Internal consistency check.
1133 */
1134static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1135{
1136 PMMHYPERCHUNKFREE pPrev = NULL;
1137 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1138 for (;;)
1139 {
1140 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1141 ASSERT_CHUNK_USED(pHeap, &pCur->core);
1142 else
1143 ASSERT_CHUNK_FREE(pHeap, pCur);
1144 if (pPrev)
1145 AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1146 ("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1147
1148# ifdef MMHYPER_HEAP_STRICT_FENCE
1149 uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
1150 if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1151 && off < pHeap->offPageAligned)
1152 {
1153 uint32_t cbCur = pCur->core.offNext
1154 ? pCur->core.offNext
1155 : pHeap->cbHeap - off;
1156 uint32_t *pu32End = ((uint32_t *)((uint8_t *)pCur + cbCur));
1157 uint32_t cbFence = pu32End[-1];
1158 if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1159 || cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1160 {
1161 mmHyperHeapDumpOne(pHeap, pCur);
1162 Assert(cbFence < cbCur - sizeof(*pCur));
1163 Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1164 }
1165
1166 uint32_t *pu32Bad = ASMMemIsAllU32((uint8_t *)pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1167 if (RT_UNLIKELY(pu32Bad))
1168 {
1169 mmHyperHeapDumpOne(pHeap, pCur);
1170 Assert(!pu32Bad);
1171 }
1172 }
1173# endif
1174
1175 /* next */
1176 if (!pCur->core.offNext)
1177 break;
1178 pPrev = pCur;
1179 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1180 }
1181}
1182#endif
1183
1184
1185/**
1186 * Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1187 * defined at build time.
1188 *
1189 * @param pVM Pointer to the VM.
1190 */
1191VMMDECL(void) MMHyperHeapCheck(PVM pVM)
1192{
1193#ifdef MMHYPER_HEAP_STRICT
1194 int rc;
1195
1196 rc = mmHyperLock(pVM);
1197 AssertRC(rc);
1198 mmHyperHeapCheck(pVM->mm.s.CTX_SUFF(pHyperHeap));
1199 mmHyperUnlock(pVM);
1200#endif
1201}
1202
1203
1204#ifdef DEBUG
1205/**
1206 * Dumps the hypervisor heap to Log.
1207 * @param pVM Pointer to the VM.
1208 */
1209VMMDECL(void) MMHyperHeapDump(PVM pVM)
1210{
1211 Log(("MMHyperHeapDump: *** heap dump - start ***\n"));
1212 PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
1213 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1214 for (;;)
1215 {
1216 mmHyperHeapDumpOne(pHeap, pCur);
1217
1218 /* next */
1219 if (!pCur->core.offNext)
1220 break;
1221 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1222 }
1223 Log(("MMHyperHeapDump: *** heap dump - end ***\n"));
1224}
1225#endif
1226
1227
1228/**
1229 * Query the amount of free memory in the hypervisor heap.
1230 *
1231 * @returns Number of free bytes in the hypervisor heap.
1232 */
1233VMMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1234{
1235 return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbFree;
1236}
1237
1238/**
1239 * Query the size the hypervisor heap.
1240 *
1241 * @returns The size of the hypervisor heap in bytes.
1242 */
1243VMMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1244{
1245 return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap;
1246}
1247
1248
1249/**
1250 * Query the address and size the hypervisor memory area.
1251 *
1252 * @returns Base address of the hypervisor area.
1253 * @param pVM Pointer to the VM.
1254 * @param pcb Where to store the size of the hypervisor area. (out)
1255 */
1256VMMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1257{
1258 if (pcb)
1259 *pcb = pVM->mm.s.cbHyperArea;
1260 return pVM->mm.s.pvHyperAreaGC;
1261}
1262
1263
1264/**
1265 * Checks if an address is within the hypervisor memory area.
1266 *
1267 * @returns true if inside.
1268 * @returns false if outside.
1269 * @param pVM Pointer to the VM.
1270 * @param GCPtr The pointer to check.
1271 */
1272VMMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1273{
1274 return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1275}
1276
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use