VirtualBox

source: vbox/trunk/src/VBox/Runtime/r3/alloc-ef.cpp@ 76553

Last change on this file since 76553 was 76553, checked in by vboxsync, 5 years ago

scm --update-copyright-year
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 31.2 KB
Line 
1/* $Id: alloc-ef.cpp 76553 2019-01-01 01:45:53Z vboxsync $ */
2/** @file
3 * IPRT - Memory Allocation, electric fence.
4 */
5
6/*
7 * Copyright (C) 2006-2019 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include "alloc-ef.h"
32#include <iprt/mem.h>
33#include <iprt/log.h>
34#include <iprt/asm.h>
35#include <iprt/thread.h>
36#include <VBox/sup.h>
37#include <iprt/errcore.h>
38#include <errno.h>
39#include <stdio.h>
40#include <stdlib.h>
41
42#include <iprt/alloc.h>
43#include <iprt/assert.h>
44#include <iprt/param.h>
45#include <iprt/string.h>
46
47#ifdef RTALLOC_REPLACE_MALLOC
48# include <VBox/dis.h>
49# include <VBox/disopcode.h>
50# include <dlfcn.h>
51# ifdef RT_OS_DARWIN
52# include <malloc/malloc.h>
53# endif
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60#ifdef RTALLOC_REPLACE_MALLOC
61# define RTMEM_REPLACMENT_ALIGN(a_cb) ((a_cb) >= 16 ? RT_ALIGN_Z(a_cb, 16) \
62 : (a_cb) >= sizeof(uintptr_t) ? RT_ALIGN_Z(a_cb, sizeof(uintptr_t)) : (a_cb))
63#endif
64
65
66/*********************************************************************************************************************************
67* Global Variables *
68*********************************************************************************************************************************/
69#ifdef RTALLOC_EFENCE_TRACE
70/** Spinlock protecting the all the block's globals. */
71static volatile uint32_t g_BlocksLock;
72/** Tree tracking the allocations. */
73static AVLPVTREE g_BlocksTree;
74# ifdef RTALLOC_EFENCE_FREE_DELAYED
75/** Tail of the delayed blocks. */
76static volatile PRTMEMBLOCK g_pBlocksDelayHead;
77/** Tail of the delayed blocks. */
78static volatile PRTMEMBLOCK g_pBlocksDelayTail;
79/** Number of bytes in the delay list (includes fences). */
80static volatile size_t g_cbBlocksDelay;
81# endif /* RTALLOC_EFENCE_FREE_DELAYED */
82# ifdef RTALLOC_REPLACE_MALLOC
83/** @name For calling the real allocation API we've replaced.
84 * @{ */
85void * (*g_pfnOrgMalloc)(size_t);
86void * (*g_pfnOrgCalloc)(size_t, size_t);
87void * (*g_pfnOrgRealloc)(void *, size_t);
88void (*g_pfnOrgFree)(void *);
89size_t (*g_pfnOrgMallocSize)(void *);
90/** @} */
91# endif
92#endif /* RTALLOC_EFENCE_TRACE */
93/** Array of pointers free watches for. */
94void *gapvRTMemFreeWatch[4] = {NULL, NULL, NULL, NULL};
95/** Enable logging of all freed memory. */
96bool gfRTMemFreeLog = false;
97
98
99/*********************************************************************************************************************************
100* Internal Functions *
101*********************************************************************************************************************************/
102#ifdef RTALLOC_REPLACE_MALLOC
103static void rtMemReplaceMallocAndFriends(void);
104#endif
105
106
107/**
108 * Complains about something.
109 */
110static void rtmemComplain(const char *pszOp, const char *pszFormat, ...)
111{
112 va_list args;
113 fprintf(stderr, "RTMem error: %s: ", pszOp);
114 va_start(args, pszFormat);
115 vfprintf(stderr, pszFormat, args);
116 va_end(args);
117 RTAssertDoPanic();
118}
119
120/**
121 * Log an event.
122 */
123DECLINLINE(void) rtmemLog(const char *pszOp, const char *pszFormat, ...)
124{
125#if 0
126 va_list args;
127 fprintf(stderr, "RTMem info: %s: ", pszOp);
128 va_start(args, pszFormat);
129 vfprintf(stderr, pszFormat, args);
130 va_end(args);
131#else
132 NOREF(pszOp); NOREF(pszFormat);
133#endif
134}
135
136
137#ifdef RTALLOC_EFENCE_TRACE
138
139/**
140 * Acquires the lock.
141 */
142DECLINLINE(void) rtmemBlockLock(void)
143{
144 unsigned c = 0;
145 while (!ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
146 RTThreadSleepNoLog(((++c) >> 2) & 31);
147}
148
149
150/**
151 * Releases the lock.
152 */
153DECLINLINE(void) rtmemBlockUnlock(void)
154{
155 Assert(g_BlocksLock == 1);
156 ASMAtomicXchgU32(&g_BlocksLock, 0);
157}
158
159
160/**
161 * Creates a block.
162 */
163DECLINLINE(PRTMEMBLOCK) rtmemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
164 const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
165{
166# ifdef RTALLOC_REPLACE_MALLOC
167 if (!g_pfnOrgMalloc)
168 rtMemReplaceMallocAndFriends();
169 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)g_pfnOrgMalloc(sizeof(*pBlock));
170# else
171 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)malloc(sizeof(*pBlock));
172# endif
173 if (pBlock)
174 {
175 pBlock->enmType = enmType;
176 pBlock->cbUnaligned = cbUnaligned;
177 pBlock->cbAligned = cbAligned;
178 pBlock->pszTag = pszTag;
179 pBlock->pvCaller = pvCaller;
180 pBlock->iLine = iLine;
181 pBlock->pszFile = pszFile;
182 pBlock->pszFunction = pszFunction;
183 }
184 return pBlock;
185}
186
187
188/**
189 * Frees a block.
190 */
191DECLINLINE(void) rtmemBlockFree(PRTMEMBLOCK pBlock)
192{
193# ifdef RTALLOC_REPLACE_MALLOC
194 g_pfnOrgFree(pBlock);
195# else
196 free(pBlock);
197# endif
198}
199
200
201/**
202 * Insert a block from the tree.
203 */
204DECLINLINE(void) rtmemBlockInsert(PRTMEMBLOCK pBlock, void *pv)
205{
206 pBlock->Core.Key = pv;
207 rtmemBlockLock();
208 bool fRc = RTAvlPVInsert(&g_BlocksTree, &pBlock->Core);
209 rtmemBlockUnlock();
210 AssertRelease(fRc);
211}
212
213
214/**
215 * Remove a block from the tree and returns it to the caller.
216 */
217DECLINLINE(PRTMEMBLOCK) rtmemBlockRemove(void *pv)
218{
219 rtmemBlockLock();
220 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVRemove(&g_BlocksTree, pv);
221 rtmemBlockUnlock();
222 return pBlock;
223}
224
225/**
226 * Gets a block.
227 */
228DECLINLINE(PRTMEMBLOCK) rtmemBlockGet(void *pv)
229{
230 rtmemBlockLock();
231 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVGet(&g_BlocksTree, pv);
232 rtmemBlockUnlock();
233 return pBlock;
234}
235
236/**
237 * Dumps one allocation.
238 */
239static DECLCALLBACK(int) RTMemDumpOne(PAVLPVNODECORE pNode, void *pvUser)
240{
241 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)pNode;
242 fprintf(stderr, "%p %08lx(+%02lx) %p\n",
243 pBlock->Core.Key,
244 (unsigned long)pBlock->cbUnaligned,
245 (unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned),
246 pBlock->pvCaller);
247 NOREF(pvUser);
248 return 0;
249}
250
251/**
252 * Dumps the allocated blocks.
253 * This is something which you should call from gdb.
254 */
255extern "C" void RTMemDump(void);
256void RTMemDump(void)
257{
258 fprintf(stderr, "address size(alg) caller\n");
259 RTAvlPVDoWithAll(&g_BlocksTree, true, RTMemDumpOne, NULL);
260}
261
262# ifdef RTALLOC_EFENCE_FREE_DELAYED
263
264/**
265 * Insert a delayed block.
266 */
267DECLINLINE(void) rtmemBlockDelayInsert(PRTMEMBLOCK pBlock)
268{
269 size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
270 pBlock->Core.pRight = NULL;
271 pBlock->Core.pLeft = NULL;
272 rtmemBlockLock();
273 if (g_pBlocksDelayHead)
274 {
275 g_pBlocksDelayHead->Core.pLeft = (PAVLPVNODECORE)pBlock;
276 pBlock->Core.pRight = (PAVLPVNODECORE)g_pBlocksDelayHead;
277 g_pBlocksDelayHead = pBlock;
278 }
279 else
280 {
281 g_pBlocksDelayTail = pBlock;
282 g_pBlocksDelayHead = pBlock;
283 }
284 g_cbBlocksDelay += cbBlock;
285 rtmemBlockUnlock();
286}
287
288/**
289 * Removes a delayed block.
290 */
291DECLINLINE(PRTMEMBLOCK) rtmemBlockDelayRemove(void)
292{
293 PRTMEMBLOCK pBlock = NULL;
294 rtmemBlockLock();
295 if (g_cbBlocksDelay > RTALLOC_EFENCE_FREE_DELAYED)
296 {
297 pBlock = g_pBlocksDelayTail;
298 if (pBlock)
299 {
300 g_pBlocksDelayTail = (PRTMEMBLOCK)pBlock->Core.pLeft;
301 if (pBlock->Core.pLeft)
302 pBlock->Core.pLeft->pRight = NULL;
303 else
304 g_pBlocksDelayHead = NULL;
305 g_cbBlocksDelay -= RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
306 }
307 }
308 rtmemBlockUnlock();
309 return pBlock;
310}
311
312# endif /* RTALLOC_EFENCE_FREE_DELAYED */
313
314#endif /* RTALLOC_EFENCE_TRACE */
315
316
317#if defined(RTALLOC_REPLACE_MALLOC) && defined(RTALLOC_EFENCE_TRACE)
318/*
319 *
320 * Replacing malloc, calloc, realloc, & free.
321 *
322 */
323
324/** Replacement for malloc. */
325static void *rtMemReplacementMalloc(size_t cb)
326{
327 size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
328 void *pv = rtR3MemAlloc("r-malloc", RTMEMTYPE_RTMEMALLOC, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
329 if (!pv)
330 pv = g_pfnOrgMalloc(cb);
331 return pv;
332}
333
334/** Replacement for calloc. */
335static void *rtMemReplacementCalloc(size_t cbItem, size_t cItems)
336{
337 size_t cb = cbItem * cItems;
338 size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
339 void *pv = rtR3MemAlloc("r-calloc", RTMEMTYPE_RTMEMALLOCZ, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
340 if (!pv)
341 pv = g_pfnOrgCalloc(cbItem, cItems);
342 return pv;
343}
344
345/** Replacement for realloc. */
346static void *rtMemReplacementRealloc(void *pvOld, size_t cbNew)
347{
348 if (pvOld)
349 {
350 /* We're not strict about where the memory was allocated. */
351 PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
352 if (pBlock)
353 {
354 size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cbNew);
355 return rtR3MemRealloc("r-realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
356 }
357 return g_pfnOrgRealloc(pvOld, cbNew);
358 }
359 return rtMemReplacementMalloc(cbNew);
360}
361
362/** Replacement for free(). */
363static void rtMemReplacementFree(void *pv)
364{
365 if (pv)
366 {
367 /* We're not strict about where the memory was allocated. */
368 PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
369 if (pBlock)
370 rtR3MemFree("r-free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS);
371 else
372 g_pfnOrgFree(pv);
373 }
374}
375
376# ifdef RT_OS_DARWIN
377/** Replacement for malloc. */
378static size_t rtMemReplacementMallocSize(void *pv)
379{
380 size_t cb;
381 if (pv)
382 {
383 /* We're not strict about where the memory was allocated. */
384 PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
385 if (pBlock)
386 cb = pBlock->cbUnaligned;
387 else
388 cb = g_pfnOrgMallocSize(pv);
389 }
390 else
391 cb = 0;
392 return cb;
393}
394# endif
395
396
397static void rtMemReplaceMallocAndFriends(void)
398{
399 struct
400 {
401 const char *pszName;
402 PFNRT pfnReplacement;
403 PFNRT pfnOrg;
404 PFNRT *ppfnJumpBack;
405 } aApis[] =
406 {
407 { "free", (PFNRT)rtMemReplacementFree, (PFNRT)free, (PFNRT *)&g_pfnOrgFree },
408 { "realloc", (PFNRT)rtMemReplacementRealloc, (PFNRT)realloc, (PFNRT *)&g_pfnOrgRealloc },
409 { "calloc", (PFNRT)rtMemReplacementCalloc, (PFNRT)calloc, (PFNRT *)&g_pfnOrgCalloc },
410 { "malloc", (PFNRT)rtMemReplacementMalloc, (PFNRT)malloc, (PFNRT *)&g_pfnOrgMalloc },
411#ifdef RT_OS_DARWIN
412 { "malloc_size", (PFNRT)rtMemReplacementMallocSize, (PFNRT)malloc_size, (PFNRT *)&g_pfnOrgMallocSize },
413#endif
414 };
415
416 /*
417 * Initialize the jump backs to avoid recursivly entering this function.
418 */
419 for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
420 *aApis[i].ppfnJumpBack = aApis[i].pfnOrg;
421
422 /*
423 * Give the user an option to skip replacing malloc.
424 */
425 if (getenv("IPRT_DONT_REPLACE_MALLOC"))
426 return;
427
428 /*
429 * Allocate a page for jump back code (we leak it).
430 */
431 uint8_t *pbExecPage = (uint8_t *)RTMemPageAlloc(PAGE_SIZE); AssertFatal(pbExecPage);
432 int rc = RTMemProtect(pbExecPage, PAGE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC); AssertFatalRC(rc);
433
434 /*
435 * Do the ground work.
436 */
437 uint8_t *pb = pbExecPage;
438 for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
439 {
440 /* Resolve it. */
441 PFNRT pfnOrg = (PFNRT)(uintptr_t)dlsym(RTLD_DEFAULT, aApis[i].pszName);
442 if (pfnOrg)
443 aApis[i].pfnOrg = pfnOrg;
444 else
445 pfnOrg = aApis[i].pfnOrg;
446
447 /* Figure what we can replace and how much to duplicate in the jump back code. */
448# ifdef RT_ARCH_AMD64
449 uint32_t cbNeeded = 12;
450 DISCPUMODE const enmCpuMode = DISCPUMODE_64BIT;
451# elif defined(RT_ARCH_X86)
452 uint32_t const cbNeeded = 5;
453 DISCPUMODE const enmCpuMode = DISCPUMODE_32BIT;
454# else
455# error "Port me"
456# endif
457 uint32_t offJmpBack = 0;
458 uint32_t cbCopy = 0;
459 while (offJmpBack < cbNeeded)
460 {
461 DISCPUSTATE Dis;
462 uint32_t cbInstr = 1;
463 rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
464 AssertFatal(!(Dis.pCurInstr->fOpType & (DISOPTYPE_CONTROLFLOW)));
465# ifdef RT_ARCH_AMD64
466# ifdef RT_OS_DARWIN
467 /* Kludge for: cmp [malloc_def_zone_state], 1; jg 2; call _malloc_initialize; 2: */
468 DISQPVPARAMVAL Parm;
469 if ( Dis.ModRM.Bits.Mod == 0
470 && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */
471 && (Dis.Param2.fUse & (DISUSE_IMMEDIATE16_SX8 | DISUSE_IMMEDIATE32_SX8 | DISUSE_IMMEDIATE64_SX8))
472 && Dis.Param2.uValue == 1
473 && Dis.pCurInstr->uOpcode == OP_CMP)
474 {
475 cbCopy = offJmpBack;
476
477 offJmpBack += cbInstr;
478 rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
479 if ( Dis.pCurInstr->uOpcode == OP_JNBE
480 && Dis.Param1.uDisp.i8 == 5)
481 {
482 offJmpBack += cbInstr + 5;
483 AssertFatal(offJmpBack >= cbNeeded);
484 break;
485 }
486 }
487# endif
488 AssertFatal(!(Dis.ModRM.Bits.Mod == 0 && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */));
489# endif
490 offJmpBack += cbInstr;
491 }
492 if (!cbCopy)
493 cbCopy = offJmpBack;
494
495 /* Assemble the jump back. */
496 memcpy(pb, (void *)(uintptr_t)pfnOrg, cbCopy);
497 uint32_t off = cbCopy;
498# ifdef RT_ARCH_AMD64
499 pb[off++] = 0xff; /* jmp qword [$+8 wrt RIP] */
500 pb[off++] = 0x25;
501 *(uint32_t *)&pb[off] = 0;
502 off += 4;
503 *(uint64_t *)&pb[off] = (uintptr_t)pfnOrg + offJmpBack;
504 off += 8;
505 off = RT_ALIGN_32(off, 16);
506# elif defined(RT_ARCH_X86)
507 pb[off++] = 0xe9; /* jmp rel32 */
508 *(uint32_t *)&pb[off] = (uintptr_t)pfnOrg + offJmpBack - (uintptr_t)&pb[4];
509 off += 4;
510 off = RT_ALIGN_32(off, 8);
511# else
512# error "Port me"
513# endif
514 *aApis[i].ppfnJumpBack = (PFNRT)(uintptr_t)pb;
515 pb += off;
516 }
517
518 /*
519 * Modify the APIs.
520 */
521 for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
522 {
523 pb = (uint8_t *)(uintptr_t)aApis[i].pfnOrg;
524 rc = RTMemProtect(pb, 16, RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC); AssertFatalRC(rc);
525
526# ifdef RT_ARCH_AMD64
527 /* Assemble the LdrLoadDll patch. */
528 *pb++ = 0x48; /* mov rax, qword */
529 *pb++ = 0xb8;
530 *(uint64_t *)pb = (uintptr_t)aApis[i].pfnReplacement;
531 pb += 8;
532 *pb++ = 0xff; /* jmp rax */
533 *pb++ = 0xe0;
534# elif defined(RT_ARCH_X86)
535 *pb++ = 0xe9; /* jmp rel32 */
536 *(uint32_t *)pb = (uintptr_t)aApis[i].pfnReplacement - (uintptr_t)&pb[4];
537# else
538# error "Port me"
539# endif
540 }
541}
542
543#endif /* RTALLOC_REPLACE_MALLOC && RTALLOC_EFENCE_TRACE */
544
545
546/**
547 * Internal allocator.
548 */
549RTDECL(void *) rtR3MemAlloc(const char *pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
550 const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
551{
552 /*
553 * Sanity.
554 */
555 if ( RT_ALIGN_Z(RTALLOC_EFENCE_SIZE, PAGE_SIZE) != RTALLOC_EFENCE_SIZE
556 && RTALLOC_EFENCE_SIZE <= 0)
557 {
558 rtmemComplain(pszOp, "Invalid E-fence size! %#x\n", RTALLOC_EFENCE_SIZE);
559 return NULL;
560 }
561 if (!cbUnaligned)
562 {
563#if 0
564 rtmemComplain(pszOp, "Request of ZERO bytes allocation!\n");
565 return NULL;
566#else
567 cbAligned = cbUnaligned = 1;
568#endif
569 }
570
571#ifndef RTALLOC_EFENCE_IN_FRONT
572 /* Alignment decreases fence accuracy, but this is at least partially
573 * counteracted by filling and checking the alignment padding. When the
574 * fence is in front then then no extra alignment is needed. */
575 cbAligned = RT_ALIGN_Z(cbAligned, RTALLOC_EFENCE_ALIGNMENT);
576#endif
577
578#ifdef RTALLOC_EFENCE_TRACE
579 /*
580 * Allocate the trace block.
581 */
582 PRTMEMBLOCK pBlock = rtmemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS);
583 if (!pBlock)
584 {
585 rtmemComplain(pszOp, "Failed to allocate trace block!\n");
586 return NULL;
587 }
588#endif
589
590 /*
591 * Allocate a block with page alignment space + the size of the E-fence.
592 */
593 size_t cbBlock = RT_ALIGN_Z(cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
594 void *pvBlock = RTMemPageAlloc(cbBlock);
595 if (pvBlock)
596 {
597 /*
598 * Calc the start of the fence and the user block
599 * and then change the page protection of the fence.
600 */
601#ifdef RTALLOC_EFENCE_IN_FRONT
602 void *pvEFence = pvBlock;
603 void *pv = (char *)pvEFence + RTALLOC_EFENCE_SIZE;
604# ifdef RTALLOC_EFENCE_NOMAN_FILLER
605 memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cbUnaligned);
606# endif
607#else
608 void *pvEFence = (char *)pvBlock + (cbBlock - RTALLOC_EFENCE_SIZE);
609 void *pv = (char *)pvEFence - cbAligned;
610# ifdef RTALLOC_EFENCE_NOMAN_FILLER
611 memset(pvBlock, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cbAligned);
612 memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbAligned - cbUnaligned);
613# endif
614#endif
615
616#ifdef RTALLOC_EFENCE_FENCE_FILLER
617 memset(pvEFence, RTALLOC_EFENCE_FENCE_FILLER, RTALLOC_EFENCE_SIZE);
618#endif
619 int rc = RTMemProtect(pvEFence, RTALLOC_EFENCE_SIZE, RTMEM_PROT_NONE);
620 if (!rc)
621 {
622#ifdef RTALLOC_EFENCE_TRACE
623 rtmemBlockInsert(pBlock, pv);
624#endif
625 if (enmType == RTMEMTYPE_RTMEMALLOCZ)
626 memset(pv, 0, cbUnaligned);
627#ifdef RTALLOC_EFENCE_FILLER
628 else
629 memset(pv, RTALLOC_EFENCE_FILLER, cbUnaligned);
630#endif
631
632 rtmemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned);
633 return pv;
634 }
635 rtmemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, RTALLOC_EFENCE_SIZE, rc);
636 RTMemPageFree(pvBlock, cbBlock);
637 }
638 else
639 rtmemComplain(pszOp, "Failed to allocated %lu (%lu) bytes.\n", (unsigned long)cbBlock, (unsigned long)cbUnaligned);
640
641#ifdef RTALLOC_EFENCE_TRACE
642 rtmemBlockFree(pBlock);
643#endif
644 return NULL;
645}
646
647
648/**
649 * Internal free.
650 */
651RTDECL(void) rtR3MemFree(const char *pszOp, RTMEMTYPE enmType, void *pv, void *pvCaller, RT_SRC_POS_DECL)
652{
653 NOREF(enmType); RT_SRC_POS_NOREF();
654
655 /*
656 * Simple case.
657 */
658 if (!pv)
659 return;
660
661 /*
662 * Check watch points.
663 */
664 for (unsigned i = 0; i < RT_ELEMENTS(gapvRTMemFreeWatch); i++)
665 if (gapvRTMemFreeWatch[i] == pv)
666 RTAssertDoPanic();
667
668#ifdef RTALLOC_EFENCE_TRACE
669 /*
670 * Find the block.
671 */
672 PRTMEMBLOCK pBlock = rtmemBlockRemove(pv);
673 if (pBlock)
674 {
675 if (gfRTMemFreeLog)
676 RTLogPrintf("RTMem %s: pv=%p pvCaller=%p cbUnaligned=%#x\n", pszOp, pv, pvCaller, pBlock->cbUnaligned);
677
678# ifdef RTALLOC_EFENCE_NOMAN_FILLER
679 /*
680 * Check whether the no man's land is untouched.
681 */
682# ifdef RTALLOC_EFENCE_IN_FRONT
683 void *pvWrong = ASMMemFirstMismatchingU8((char *)pv + pBlock->cbUnaligned,
684 RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbUnaligned,
685 RTALLOC_EFENCE_NOMAN_FILLER);
686# else
687 /* Alignment must match allocation alignment in rtMemAlloc(). */
688 void *pvWrong = ASMMemFirstMismatchingU8((char *)pv + pBlock->cbUnaligned,
689 pBlock->cbAligned - pBlock->cbUnaligned,
690 RTALLOC_EFENCE_NOMAN_FILLER);
691 if (pvWrong)
692 RTAssertDoPanic();
693 pvWrong = ASMMemFirstMismatchingU8((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK),
694 RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbAligned,
695 RTALLOC_EFENCE_NOMAN_FILLER);
696# endif
697 if (pvWrong)
698 RTAssertDoPanic();
699# endif
700
701# ifdef RTALLOC_EFENCE_FREE_FILL
702 /*
703 * Fill the user part of the block.
704 */
705 memset(pv, RTALLOC_EFENCE_FREE_FILL, pBlock->cbUnaligned);
706# endif
707
708# if defined(RTALLOC_EFENCE_FREE_DELAYED) && RTALLOC_EFENCE_FREE_DELAYED > 0
709 /*
710 * We're doing delayed freeing.
711 * That means we'll expand the E-fence to cover the entire block.
712 */
713 int rc = RTMemProtect(pv, pBlock->cbAligned, RTMEM_PROT_NONE);
714 if (RT_SUCCESS(rc))
715 {
716 /*
717 * Insert it into the free list and process pending frees.
718 */
719 rtmemBlockDelayInsert(pBlock);
720 while ((pBlock = rtmemBlockDelayRemove()) != NULL)
721 {
722 pv = pBlock->Core.Key;
723# ifdef RTALLOC_EFENCE_IN_FRONT
724 void *pvBlock = (char *)pv - RTALLOC_EFENCE_SIZE;
725# else
726 void *pvBlock = (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
727# endif
728 size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
729 rc = RTMemProtect(pvBlock, cbBlock, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
730 if (RT_SUCCESS(rc))
731 RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE);
732 else
733 rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ | RTMEM_PROT_WRITE) -> %d\n", pvBlock, cbBlock, rc);
734 rtmemBlockFree(pBlock);
735 }
736 }
737 else
738 rtmemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc);
739
740# else /* !RTALLOC_EFENCE_FREE_DELAYED */
741
742 /*
743 * Turn of the E-fence and free it.
744 */
745# ifdef RTALLOC_EFENCE_IN_FRONT
746 void *pvBlock = (char *)pv - RTALLOC_EFENCE_SIZE;
747 void *pvEFence = pvBlock;
748# else
749 void *pvBlock = (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
750 void *pvEFence = (char *)pv + pBlock->cb;
751# endif
752 int rc = RTMemProtect(pvEFence, RTALLOC_EFENCE_SIZE, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
753 if (RT_SUCCESS(rc))
754 RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE);
755 else
756 rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ | RTMEM_PROT_WRITE) -> %d\n", pvEFence, RTALLOC_EFENCE_SIZE, rc);
757 rtmemBlockFree(pBlock);
758
759# endif /* !RTALLOC_EFENCE_FREE_DELAYED */
760 }
761 else
762 rtmemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv);
763
764#else /* !RTALLOC_EFENCE_TRACE */
765
766 /*
767 * We have no size tracking, so we're not doing any freeing because
768 * we cannot if the E-fence is after the block.
769 * Let's just expand the E-fence to the first page of the user bit
770 * since we know that it's around.
771 */
772 int rc = RTMemProtect((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK), PAGE_SIZE, RTMEM_PROT_NONE);
773 if (RT_FAILURE(rc))
774 rtmemComplain(pszOp, "RTMemProtect(%p, PAGE_SIZE, RTMEM_PROT_NONE) -> %d\n", (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK), rc);
775#endif /* !RTALLOC_EFENCE_TRACE */
776}
777
778
779/**
780 * Internal realloc.
781 */
782RTDECL(void *) rtR3MemRealloc(const char *pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew,
783 const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
784{
785 /*
786 * Allocate new and copy.
787 */
788 if (!pvOld)
789 return rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
790 if (!cbNew)
791 {
792 rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
793 return NULL;
794 }
795
796#ifdef RTALLOC_EFENCE_TRACE
797
798 /*
799 * Get the block, allocate the new, copy the data, free the old one.
800 */
801 PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
802 if (pBlock)
803 {
804 void *pvRet = rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
805 if (pvRet)
806 {
807 memcpy(pvRet, pvOld, RT_MIN(cbNew, pBlock->cbUnaligned));
808 rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
809 }
810 return pvRet;
811 }
812 else
813 rtmemComplain(pszOp, "pvOld=%p was not found!\n", pvOld);
814 return NULL;
815
816#else /* !RTALLOC_EFENCE_TRACE */
817
818 rtmemComplain(pszOp, "Not supported if RTALLOC_EFENCE_TRACE isn't defined!\n");
819 return NULL;
820
821#endif /* !RTALLOC_EFENCE_TRACE */
822}
823
824
825
826
827RTDECL(void *) RTMemEfTmpAlloc(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
828{
829 return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
830}
831
832
833RTDECL(void *) RTMemEfTmpAllocZ(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
834{
835 return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
836}
837
838
839RTDECL(void) RTMemEfTmpFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
840{
841 if (pv)
842 rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
843}
844
845
846RTDECL(void *) RTMemEfAlloc(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
847{
848 return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
849}
850
851
852RTDECL(void *) RTMemEfAllocZ(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
853{
854 return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
855}
856
857
858RTDECL(void *) RTMemEfAllocVar(size_t cbUnaligned, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
859{
860 size_t cbAligned;
861 if (cbUnaligned >= 16)
862 cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
863 else
864 cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
865 return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
866}
867
868
869RTDECL(void *) RTMemEfAllocZVar(size_t cbUnaligned, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
870{
871 size_t cbAligned;
872 if (cbUnaligned >= 16)
873 cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
874 else
875 cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
876 return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
877}
878
879
880RTDECL(void *) RTMemEfRealloc(void *pvOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
881{
882 return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
883}
884
885
886RTDECL(void) RTMemEfFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
887{
888 if (pv)
889 rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
890}
891
892
893RTDECL(void *) RTMemEfDup(const void *pvSrc, size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
894{
895 void *pvDst = RTMemEfAlloc(cb, pszTag, RT_SRC_POS_ARGS);
896 if (pvDst)
897 memcpy(pvDst, pvSrc, cb);
898 return pvDst;
899}
900
901
902RTDECL(void *) RTMemEfDupEx(const void *pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
903{
904 void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, RT_SRC_POS_ARGS);
905 if (pvDst)
906 {
907 memcpy(pvDst, pvSrc, cbSrc);
908 memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
909 }
910 return pvDst;
911}
912
913
914
915
916/*
917 *
918 * The NP (no position) versions.
919 *
920 */
921
922
923
924RTDECL(void *) RTMemEfTmpAllocNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
925{
926 return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
927}
928
929
930RTDECL(void *) RTMemEfTmpAllocZNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
931{
932 return rtR3MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
933}
934
935
936RTDECL(void) RTMemEfTmpFreeNP(void *pv) RT_NO_THROW_DEF
937{
938 if (pv)
939 rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
940}
941
942
943RTDECL(void *) RTMemEfAllocNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
944{
945 return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
946}
947
948
949RTDECL(void *) RTMemEfAllocZNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
950{
951 return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
952}
953
954
955RTDECL(void *) RTMemEfAllocVarNP(size_t cbUnaligned, const char *pszTag) RT_NO_THROW_DEF
956{
957 size_t cbAligned;
958 if (cbUnaligned >= 16)
959 cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
960 else
961 cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
962 return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
963}
964
965
966RTDECL(void *) RTMemEfAllocZVarNP(size_t cbUnaligned, const char *pszTag) RT_NO_THROW_DEF
967{
968 size_t cbAligned;
969 if (cbUnaligned >= 16)
970 cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
971 else
972 cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
973 return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
974}
975
976
977RTDECL(void *) RTMemEfReallocNP(void *pvOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
978{
979 return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
980}
981
982
983RTDECL(void) RTMemEfFreeNP(void *pv) RT_NO_THROW_DEF
984{
985 if (pv)
986 rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
987}
988
989
990RTDECL(void *) RTMemEfDupNP(const void *pvSrc, size_t cb, const char *pszTag) RT_NO_THROW_DEF
991{
992 void *pvDst = RTMemEfAlloc(cb, pszTag, NULL, 0, NULL);
993 if (pvDst)
994 memcpy(pvDst, pvSrc, cb);
995 return pvDst;
996}
997
998
999RTDECL(void *) RTMemEfDupExNP(const void *pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag) RT_NO_THROW_DEF
1000{
1001 void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, NULL, 0, NULL);
1002 if (pvDst)
1003 {
1004 memcpy(pvDst, pvSrc, cbSrc);
1005 memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
1006 }
1007 return pvDst;
1008}
1009
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use