VirtualBox

source: vbox/trunk/src/VBox/Runtime/r3/alloc-ef.cpp

Last change on this file was 103005, checked in by vboxsync, 4 months ago

iprt/asm.h,*: Split out the ASMMem* and related stuff into a separate header, asm-mem.h, so that we can get the RT_ASM_PAGE_SIZE stuff out of the way.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 33.9 KB
Line 
1/* $Id: alloc-ef.cpp 103005 2024-01-23 23:55:58Z vboxsync $ */
2/** @file
3 * IPRT - Memory Allocation, electric fence.
4 */
5
6/*
7 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * The contents of this file may alternatively be used under the terms
26 * of the Common Development and Distribution License Version 1.0
27 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28 * in the VirtualBox distribution, in which case the provisions of the
29 * CDDL are applicable instead of those of the GPL.
30 *
31 * You may elect to license modified versions of this file under the
32 * terms and conditions of either the GPL or the CDDL or both.
33 *
34 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35 */
36
37
38/*********************************************************************************************************************************
39* Header Files *
40*********************************************************************************************************************************/
41#include "alloc-ef.h"
42#include <iprt/mem.h>
43#include <iprt/log.h>
44#include <iprt/asm.h>
45#include <iprt/asm-mem.h>
46#include <iprt/thread.h>
47#include <VBox/sup.h>
48#include <iprt/errcore.h>
49#ifndef IPRT_NO_CRT
50# include <errno.h>
51# include <stdio.h>
52# include <stdlib.h>
53#endif
54
55#include <iprt/alloc.h>
56#include <iprt/assert.h>
57#include <iprt/param.h>
58#include <iprt/string.h>
59#include <iprt/system.h>
60
61#ifdef RTALLOC_REPLACE_MALLOC
62# include <VBox/dis.h>
63# include <VBox/disopcode.h>
64# include <dlfcn.h>
65# ifdef RT_OS_DARWIN
66# include <malloc/malloc.h>
67# endif
68#endif
69
70
71/*********************************************************************************************************************************
72* Defined Constants And Macros *
73*********************************************************************************************************************************/
74#ifdef RTALLOC_REPLACE_MALLOC
75# define RTMEM_REPLACMENT_ALIGN(a_cb) ((a_cb) >= 16 ? RT_ALIGN_Z(a_cb, 16) \
76 : (a_cb) >= sizeof(uintptr_t) ? RT_ALIGN_Z(a_cb, sizeof(uintptr_t)) : (a_cb))
77#endif
78
79
80/*********************************************************************************************************************************
81* Global Variables *
82*********************************************************************************************************************************/
83#ifdef RTALLOC_EFENCE_TRACE
84/** Spinlock protecting the all the block's globals. */
85static volatile uint32_t g_BlocksLock;
86/** Tree tracking the allocations. */
87static AVLPVTREE g_BlocksTree;
88# ifdef RTALLOC_EFENCE_FREE_DELAYED
89/** Tail of the delayed blocks. */
90static volatile PRTMEMBLOCK g_pBlocksDelayHead;
91/** Tail of the delayed blocks. */
92static volatile PRTMEMBLOCK g_pBlocksDelayTail;
93/** Number of bytes in the delay list (includes fences). */
94static volatile size_t g_cbBlocksDelay;
95# endif /* RTALLOC_EFENCE_FREE_DELAYED */
96# ifdef RTALLOC_REPLACE_MALLOC
97/** @name For calling the real allocation API we've replaced.
98 * @{ */
99void * (*g_pfnOrgMalloc)(size_t);
100void * (*g_pfnOrgCalloc)(size_t, size_t);
101void * (*g_pfnOrgRealloc)(void *, size_t);
102void (*g_pfnOrgFree)(void *);
103size_t (*g_pfnOrgMallocSize)(void *);
104/** @} */
105# endif
106#endif /* RTALLOC_EFENCE_TRACE */
107/** Array of pointers free watches for. */
108void *gapvRTMemFreeWatch[4] = {NULL, NULL, NULL, NULL};
109/** Enable logging of all freed memory. */
110bool gfRTMemFreeLog = false;
111
112
113/*********************************************************************************************************************************
114* Internal Functions *
115*********************************************************************************************************************************/
116#ifdef RTALLOC_REPLACE_MALLOC
117static void rtMemReplaceMallocAndFriends(void);
118#endif
119
120
121/**
122 * Complains about something.
123 */
124static void rtmemComplain(const char *pszOp, const char *pszFormat, ...)
125{
126 va_list args;
127 fprintf(stderr, "RTMem error: %s: ", pszOp);
128 va_start(args, pszFormat);
129 vfprintf(stderr, pszFormat, args);
130 va_end(args);
131 RTAssertDoPanic();
132}
133
134/**
135 * Log an event.
136 */
137DECLINLINE(void) rtmemLog(const char *pszOp, const char *pszFormat, ...)
138{
139#if 0
140 va_list args;
141 fprintf(stderr, "RTMem info: %s: ", pszOp);
142 va_start(args, pszFormat);
143 vfprintf(stderr, pszFormat, args);
144 va_end(args);
145#else
146 NOREF(pszOp); NOREF(pszFormat);
147#endif
148}
149
150
151#ifdef RTALLOC_EFENCE_TRACE
152
153/**
154 * Acquires the lock.
155 */
156DECLINLINE(void) rtmemBlockLock(void)
157{
158 unsigned c = 0;
159 while (!ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
160 RTThreadSleepNoLog(((++c) >> 2) & 31);
161}
162
163
164/**
165 * Releases the lock.
166 */
167DECLINLINE(void) rtmemBlockUnlock(void)
168{
169 Assert(g_BlocksLock == 1);
170 ASMAtomicXchgU32(&g_BlocksLock, 0);
171}
172
173
174/**
175 * Creates a block.
176 */
177DECLINLINE(PRTMEMBLOCK) rtmemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
178 const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
179{
180# ifdef RTALLOC_REPLACE_MALLOC
181 if (!g_pfnOrgMalloc)
182 rtMemReplaceMallocAndFriends();
183 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)g_pfnOrgMalloc(sizeof(*pBlock));
184# else
185 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)malloc(sizeof(*pBlock));
186# endif
187 if (pBlock)
188 {
189 pBlock->enmType = enmType;
190 pBlock->cbUnaligned = cbUnaligned;
191 pBlock->cbAligned = cbAligned;
192 pBlock->pszTag = pszTag;
193 pBlock->pvCaller = pvCaller;
194 pBlock->iLine = iLine;
195 pBlock->pszFile = pszFile;
196 pBlock->pszFunction = pszFunction;
197 }
198 return pBlock;
199}
200
201
202/**
203 * Frees a block.
204 */
205DECLINLINE(void) rtmemBlockFree(PRTMEMBLOCK pBlock)
206{
207# ifdef RTALLOC_REPLACE_MALLOC
208 g_pfnOrgFree(pBlock);
209# else
210 free(pBlock);
211# endif
212}
213
214
215/**
216 * Insert a block from the tree.
217 */
218DECLINLINE(void) rtmemBlockInsert(PRTMEMBLOCK pBlock, void *pv)
219{
220 pBlock->Core.Key = pv;
221 rtmemBlockLock();
222 bool fRc = RTAvlPVInsert(&g_BlocksTree, &pBlock->Core);
223 rtmemBlockUnlock();
224 AssertRelease(fRc);
225}
226
227
228/**
229 * Remove a block from the tree and returns it to the caller.
230 */
231DECLINLINE(PRTMEMBLOCK) rtmemBlockRemove(void *pv)
232{
233 rtmemBlockLock();
234 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVRemove(&g_BlocksTree, pv);
235 rtmemBlockUnlock();
236 return pBlock;
237}
238
239/**
240 * Gets a block.
241 */
242DECLINLINE(PRTMEMBLOCK) rtmemBlockGet(void *pv)
243{
244 rtmemBlockLock();
245 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVGet(&g_BlocksTree, pv);
246 rtmemBlockUnlock();
247 return pBlock;
248}
249
250/**
251 * Dumps one allocation.
252 */
253static DECLCALLBACK(int) RTMemDumpOne(PAVLPVNODECORE pNode, void *pvUser)
254{
255 PRTMEMBLOCK pBlock = (PRTMEMBLOCK)pNode;
256 fprintf(stderr, "%p %08lx(+%02lx) %p\n",
257 pBlock->Core.Key,
258 (unsigned long)pBlock->cbUnaligned,
259 (unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned),
260 pBlock->pvCaller);
261 NOREF(pvUser);
262 return 0;
263}
264
265/**
266 * Dumps the allocated blocks.
267 * This is something which you should call from gdb.
268 */
269extern "C" void RTMemDump(void);
270void RTMemDump(void)
271{
272 fprintf(stderr, "address size(alg) caller\n");
273 RTAvlPVDoWithAll(&g_BlocksTree, true, RTMemDumpOne, NULL);
274}
275
276# ifdef RTALLOC_EFENCE_FREE_DELAYED
277
278/**
279 * Insert a delayed block.
280 */
281DECLINLINE(void) rtmemBlockDelayInsert(PRTMEMBLOCK pBlock)
282{
283 size_t const cbFence = RTALLOC_EFENCE_SIZE_FACTOR * RTSystemGetPageSize();
284 size_t const cbBlock = RTSystemPageAlignSize(pBlock->cbAligned) + cbFence;
285 pBlock->Core.pRight = NULL;
286 pBlock->Core.pLeft = NULL;
287 rtmemBlockLock();
288 if (g_pBlocksDelayHead)
289 {
290 g_pBlocksDelayHead->Core.pLeft = (PAVLPVNODECORE)pBlock;
291 pBlock->Core.pRight = (PAVLPVNODECORE)g_pBlocksDelayHead;
292 g_pBlocksDelayHead = pBlock;
293 }
294 else
295 {
296 g_pBlocksDelayTail = pBlock;
297 g_pBlocksDelayHead = pBlock;
298 }
299 g_cbBlocksDelay += cbBlock;
300 rtmemBlockUnlock();
301}
302
303/**
304 * Removes a delayed block.
305 */
306DECLINLINE(PRTMEMBLOCK) rtmemBlockDelayRemove(void)
307{
308 PRTMEMBLOCK pBlock = NULL;
309 rtmemBlockLock();
310 if (g_cbBlocksDelay > RTALLOC_EFENCE_FREE_DELAYED)
311 {
312 pBlock = g_pBlocksDelayTail;
313 if (pBlock)
314 {
315 g_pBlocksDelayTail = (PRTMEMBLOCK)pBlock->Core.pLeft;
316 if (pBlock->Core.pLeft)
317 pBlock->Core.pLeft->pRight = NULL;
318 else
319 g_pBlocksDelayHead = NULL;
320
321 size_t const cbFence = RTALLOC_EFENCE_SIZE_FACTOR * RTSystemGetPageSize();
322 g_cbBlocksDelay -= RTSystemPageAlignSize(pBlock->cbAligned) + cbFence;
323 }
324 }
325 rtmemBlockUnlock();
326 return pBlock;
327}
328
329
330/**
331 * Dumps the freed blocks.
332 * This is something which you should call from gdb.
333 */
334extern "C" void RTMemDumpFreed(void);
335void RTMemDumpFreed(void)
336{
337 fprintf(stderr, "address size(alg) caller\n");
338 for (PRTMEMBLOCK pCur = g_pBlocksDelayHead; pCur; pCur = (PRTMEMBLOCK)pCur->Core.pRight)
339 RTMemDumpOne(&pCur->Core, NULL);
340
341}
342
343# endif /* RTALLOC_EFENCE_FREE_DELAYED */
344
345#endif /* RTALLOC_EFENCE_TRACE */
346
347
348#if defined(RTALLOC_REPLACE_MALLOC) && defined(RTALLOC_EFENCE_TRACE)
349/*
350 *
351 * Replacing malloc, calloc, realloc, & free.
352 *
353 */
354
355/** Replacement for malloc. */
356static void *rtMemReplacementMalloc(size_t cb)
357{
358 size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
359 void *pv = rtR3MemAlloc("r-malloc", RTMEMTYPE_RTMEMALLOC, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
360 if (!pv)
361 pv = g_pfnOrgMalloc(cb);
362 return pv;
363}
364
365/** Replacement for calloc. */
366static void *rtMemReplacementCalloc(size_t cbItem, size_t cItems)
367{
368 size_t cb = cbItem * cItems;
369 size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
370 void *pv = rtR3MemAlloc("r-calloc", RTMEMTYPE_RTMEMALLOCZ, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
371 if (!pv)
372 pv = g_pfnOrgCalloc(cbItem, cItems);
373 return pv;
374}
375
376/** Replacement for realloc. */
377static void *rtMemReplacementRealloc(void *pvOld, size_t cbNew)
378{
379 if (pvOld)
380 {
381 /* We're not strict about where the memory was allocated. */
382 PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
383 if (pBlock)
384 {
385 size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cbNew);
386 return rtR3MemRealloc("r-realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
387 }
388 return g_pfnOrgRealloc(pvOld, cbNew);
389 }
390 return rtMemReplacementMalloc(cbNew);
391}
392
393/** Replacement for free(). */
394static void rtMemReplacementFree(void *pv)
395{
396 if (pv)
397 {
398 /* We're not strict about where the memory was allocated. */
399 PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
400 if (pBlock)
401 rtR3MemFree("r-free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS);
402 else
403 g_pfnOrgFree(pv);
404 }
405}
406
407# ifdef RT_OS_DARWIN
408/** Replacement for malloc. */
409static size_t rtMemReplacementMallocSize(void *pv)
410{
411 size_t cb;
412 if (pv)
413 {
414 /* We're not strict about where the memory was allocated. */
415 PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
416 if (pBlock)
417 cb = pBlock->cbUnaligned;
418 else
419 cb = g_pfnOrgMallocSize(pv);
420 }
421 else
422 cb = 0;
423 return cb;
424}
425# endif
426
427
428static void rtMemReplaceMallocAndFriends(void)
429{
430 struct
431 {
432 const char *pszName;
433 PFNRT pfnReplacement;
434 PFNRT pfnOrg;
435 PFNRT *ppfnJumpBack;
436 } aApis[] =
437 {
438 { "free", (PFNRT)rtMemReplacementFree, (PFNRT)free, (PFNRT *)&g_pfnOrgFree },
439 { "realloc", (PFNRT)rtMemReplacementRealloc, (PFNRT)realloc, (PFNRT *)&g_pfnOrgRealloc },
440 { "calloc", (PFNRT)rtMemReplacementCalloc, (PFNRT)calloc, (PFNRT *)&g_pfnOrgCalloc },
441 { "malloc", (PFNRT)rtMemReplacementMalloc, (PFNRT)malloc, (PFNRT *)&g_pfnOrgMalloc },
442#ifdef RT_OS_DARWIN
443 { "malloc_size", (PFNRT)rtMemReplacementMallocSize, (PFNRT)malloc_size, (PFNRT *)&g_pfnOrgMallocSize },
444#endif
445 };
446
447 /*
448 * Initialize the jump backs to avoid recursivly entering this function.
449 */
450 for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
451 *aApis[i].ppfnJumpBack = aApis[i].pfnOrg;
452
453 /*
454 * Give the user an option to skip replacing malloc.
455 */
456 if (getenv("IPRT_DONT_REPLACE_MALLOC"))
457 return;
458
459 /*
460 * Allocate a page for jump back code (we leak it).
461 */
462 size_t const cbPage = RTSystemGetPageSize();
463 uint8_t *pbExecPage = (uint8_t *)RTMemPageAlloc(cbPage); AssertFatal(pbExecPage);
464 int rc = RTMemProtect(pbExecPage, cbPage, RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC); AssertFatalRC(rc);
465
466 /*
467 * Do the ground work.
468 */
469 uint8_t *pb = pbExecPage;
470 for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
471 {
472 /* Resolve it. */
473 PFNRT pfnOrg = (PFNRT)(uintptr_t)dlsym(RTLD_DEFAULT, aApis[i].pszName);
474 if (pfnOrg)
475 aApis[i].pfnOrg = pfnOrg;
476 else
477 pfnOrg = aApis[i].pfnOrg;
478
479 /* Figure what we can replace and how much to duplicate in the jump back code. */
480# ifdef RT_ARCH_AMD64
481 uint32_t cbNeeded = 12;
482 DISCPUMODE const enmCpuMode = DISCPUMODE_64BIT;
483# elif defined(RT_ARCH_X86)
484 uint32_t const cbNeeded = 5;
485 DISCPUMODE const enmCpuMode = DISCPUMODE_32BIT;
486# else
487# error "Port me"
488# endif
489 uint32_t offJmpBack = 0;
490 uint32_t cbCopy = 0;
491 while (offJmpBack < cbNeeded)
492 {
493 DISCPUSTATE Dis;
494 uint32_t cbInstr = 1;
495 rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
496 AssertFatal(!(Dis.pCurInstr->fOpType & (DISOPTYPE_CONTROLFLOW)));
497# ifdef RT_ARCH_AMD64
498# ifdef RT_OS_DARWIN
499 /* Kludge for: cmp [malloc_def_zone_state], 1; jg 2; call _malloc_initialize; 2: */
500 if ( Dis.ModRM.Bits.Mod == 0
501 && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */
502 && (Dis.Param2.fUse & (DISUSE_IMMEDIATE16_SX8 | DISUSE_IMMEDIATE32_SX8 | DISUSE_IMMEDIATE64_SX8))
503 && Dis.Param2.uValue == 1
504 && Dis.pCurInstr->uOpcode == OP_CMP)
505 {
506 cbCopy = offJmpBack;
507
508 offJmpBack += cbInstr;
509 rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
510 if ( Dis.pCurInstr->uOpcode == OP_JNBE
511 && Dis.Param1.uDisp.i8 == 5)
512 {
513 offJmpBack += cbInstr + 5;
514 AssertFatal(offJmpBack >= cbNeeded);
515 break;
516 }
517 }
518# endif
519 AssertFatal(!(Dis.ModRM.Bits.Mod == 0 && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */));
520# endif
521 offJmpBack += cbInstr;
522 }
523 if (!cbCopy)
524 cbCopy = offJmpBack;
525
526 /* Assemble the jump back. */
527 memcpy(pb, (void *)(uintptr_t)pfnOrg, cbCopy);
528 uint32_t off = cbCopy;
529# ifdef RT_ARCH_AMD64
530 pb[off++] = 0xff; /* jmp qword [$+8 wrt RIP] */
531 pb[off++] = 0x25;
532 *(uint32_t *)&pb[off] = 0;
533 off += 4;
534 *(uint64_t *)&pb[off] = (uintptr_t)pfnOrg + offJmpBack;
535 off += 8;
536 off = RT_ALIGN_32(off, 16);
537# elif defined(RT_ARCH_X86)
538 pb[off++] = 0xe9; /* jmp rel32 */
539 *(uint32_t *)&pb[off] = (uintptr_t)pfnOrg + offJmpBack - (uintptr_t)&pb[4];
540 off += 4;
541 off = RT_ALIGN_32(off, 8);
542# else
543# error "Port me"
544# endif
545 *aApis[i].ppfnJumpBack = (PFNRT)(uintptr_t)pb;
546 pb += off;
547 }
548
549 /*
550 * Modify the APIs.
551 */
552 for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
553 {
554 pb = (uint8_t *)(uintptr_t)aApis[i].pfnOrg;
555 rc = RTMemProtect(pb, 16, RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC); AssertFatalRC(rc);
556
557# ifdef RT_ARCH_AMD64
558 /* Assemble the LdrLoadDll patch. */
559 *pb++ = 0x48; /* mov rax, qword */
560 *pb++ = 0xb8;
561 *(uint64_t *)pb = (uintptr_t)aApis[i].pfnReplacement;
562 pb += 8;
563 *pb++ = 0xff; /* jmp rax */
564 *pb++ = 0xe0;
565# elif defined(RT_ARCH_X86)
566 *pb++ = 0xe9; /* jmp rel32 */
567 *(uint32_t *)pb = (uintptr_t)aApis[i].pfnReplacement - (uintptr_t)&pb[4];
568# else
569# error "Port me"
570# endif
571 }
572}
573
574#endif /* RTALLOC_REPLACE_MALLOC && RTALLOC_EFENCE_TRACE */
575
576
577/**
578 * Internal allocator.
579 */
580RTDECL(void *) rtR3MemAlloc(const char *pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
581 const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
582{
583 /*
584 * Sanity.
585 */
586 size_t const cbFence = RTALLOC_EFENCE_SIZE_FACTOR * RTSystemGetPageSize();
587 size_t const cbPage = RTSystemGetPageSize();
588 if (RTALLOC_EFENCE_SIZE_FACTOR <= 0)
589 {
590 rtmemComplain(pszOp, "Invalid E-fence size! %#x\n", RTALLOC_EFENCE_SIZE_FACTOR);
591 return NULL;
592 }
593 if (!cbUnaligned)
594 {
595#if 0
596 rtmemComplain(pszOp, "Request of ZERO bytes allocation!\n");
597 return NULL;
598#else
599 cbAligned = cbUnaligned = 1;
600#endif
601 }
602
603#ifndef RTALLOC_EFENCE_IN_FRONT
604 /* Alignment decreases fence accuracy, but this is at least partially
605 * counteracted by filling and checking the alignment padding. When the
606 * fence is in front then then no extra alignment is needed. */
607 cbAligned = RT_ALIGN_Z(cbAligned, RTALLOC_EFENCE_ALIGNMENT);
608#endif
609
610#ifdef RTALLOC_EFENCE_TRACE
611 /*
612 * Allocate the trace block.
613 */
614 PRTMEMBLOCK pBlock = rtmemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS);
615 if (!pBlock)
616 {
617 rtmemComplain(pszOp, "Failed to allocate trace block!\n");
618 return NULL;
619 }
620#endif
621
622 /*
623 * Allocate a block with page alignment space + the size of the E-fence.
624 */
625 size_t cbBlock = RT_ALIGN_Z(cbAligned, cbPage) + cbFence;
626 void *pvBlock = RTMemPageAlloc(cbBlock);
627 if (pvBlock)
628 {
629 /*
630 * Calc the start of the fence and the user block
631 * and then change the page protection of the fence.
632 */
633#ifdef RTALLOC_EFENCE_IN_FRONT
634 void *pvEFence = pvBlock;
635 void *pv = (char *)pvEFence + cbFence;
636# ifdef RTALLOC_EFENCE_NOMAN_FILLER
637 memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - cbFence - cbUnaligned);
638# endif
639#else
640 void *pvEFence = (char *)pvBlock + (cbBlock - cbFence);
641 void *pv = (char *)pvEFence - cbAligned;
642# ifdef RTALLOC_EFENCE_NOMAN_FILLER
643 memset(pvBlock, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - cbFence - cbAligned);
644 memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbAligned - cbUnaligned);
645# endif
646#endif
647
648#ifdef RTALLOC_EFENCE_FENCE_FILLER
649 memset(pvEFence, RTALLOC_EFENCE_FENCE_FILLER, cbFence);
650#endif
651 int rc = RTMemProtect(pvEFence, cbFence, RTMEM_PROT_NONE);
652 if (!rc)
653 {
654#ifdef RTALLOC_EFENCE_TRACE
655 rtmemBlockInsert(pBlock, pv);
656#endif
657 if (enmType == RTMEMTYPE_RTMEMALLOCZ)
658 memset(pv, 0, cbUnaligned);
659#ifdef RTALLOC_EFENCE_FILLER
660 else
661 memset(pv, RTALLOC_EFENCE_FILLER, cbUnaligned);
662#endif
663
664 rtmemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned);
665 return pv;
666 }
667 rtmemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, cbFence, rc);
668 RTMemPageFree(pvBlock, cbBlock);
669 }
670 else
671 rtmemComplain(pszOp, "Failed to allocated %lu (%lu) bytes.\n", (unsigned long)cbBlock, (unsigned long)cbUnaligned);
672
673#ifdef RTALLOC_EFENCE_TRACE
674 rtmemBlockFree(pBlock);
675#endif
676 return NULL;
677}
678
679
680/**
681 * Internal free.
682 */
683RTDECL(void) rtR3MemFree(const char *pszOp, RTMEMTYPE enmType, void *pv, size_t cbUser, void *pvCaller, RT_SRC_POS_DECL)
684{
685 NOREF(enmType); RT_SRC_POS_NOREF();
686
687 /*
688 * Simple case.
689 */
690 if (!pv)
691 return;
692
693 /*
694 * Check watch points.
695 */
696 for (unsigned i = 0; i < RT_ELEMENTS(gapvRTMemFreeWatch); i++)
697 if (gapvRTMemFreeWatch[i] == pv)
698 RTAssertDoPanic();
699
700 size_t cbPage = RTSystemGetPageSize();
701#ifdef RTALLOC_EFENCE_TRACE
702 /*
703 * Find the block.
704 */
705 PRTMEMBLOCK pBlock = rtmemBlockRemove(pv);
706 if (pBlock)
707 {
708 if (gfRTMemFreeLog)
709 RTLogPrintf("RTMem %s: pv=%p pvCaller=%p cbUnaligned=%#x\n", pszOp, pv, pvCaller, pBlock->cbUnaligned);
710
711# ifdef RTALLOC_EFENCE_NOMAN_FILLER
712 /*
713 * Check whether the no man's land is untouched.
714 */
715# ifdef RTALLOC_EFENCE_IN_FRONT
716 void *pvWrong = ASMMemFirstMismatchingU8((char *)pv + pBlock->cbUnaligned,
717 RT_ALIGN_Z(pBlock->cbAligned, cbPage) - pBlock->cbUnaligned,
718 RTALLOC_EFENCE_NOMAN_FILLER);
719# else
720 /* Alignment must match allocation alignment in rtMemAlloc(). */
721 void *pvWrong = ASMMemFirstMismatchingU8((char *)pv + pBlock->cbUnaligned,
722 pBlock->cbAligned - pBlock->cbUnaligned,
723 RTALLOC_EFENCE_NOMAN_FILLER);
724 if (pvWrong)
725 RTAssertDoPanic();
726 pvWrong = ASMMemFirstMismatchingU8((void *)((uintptr_t)pv & ~RTSystemGetPageOffsetMask()),
727 RT_ALIGN_Z(pBlock->cbAligned, cbPage) - pBlock->cbAligned,
728 RTALLOC_EFENCE_NOMAN_FILLER);
729# endif
730 if (pvWrong)
731 RTAssertDoPanic();
732# endif
733
734 /*
735 * Fill the user part of the block.
736 */
737 AssertMsg(enmType != RTMEMTYPE_RTMEMFREEZ || cbUser == pBlock->cbUnaligned,
738 ("cbUser=%#zx cbUnaligned=%#zx\n", cbUser, pBlock->cbUnaligned));
739 RT_NOREF(cbUser);
740 if (enmType == RTMEMTYPE_RTMEMFREEZ)
741 RT_BZERO(pv, pBlock->cbUnaligned);
742# ifdef RTALLOC_EFENCE_FREE_FILL
743 else
744 memset(pv, RTALLOC_EFENCE_FREE_FILL, pBlock->cbUnaligned);
745# endif
746
747 size_t const cbFence = RTALLOC_EFENCE_SIZE_FACTOR * RTSystemGetPageSize();
748# if defined(RTALLOC_EFENCE_FREE_DELAYED) && RTALLOC_EFENCE_FREE_DELAYED > 0
749 /*
750 * We're doing delayed freeing.
751 * That means we'll expand the E-fence to cover the entire block.
752 */
753 int rc = RTMemProtect(pv, pBlock->cbAligned, RTMEM_PROT_NONE);
754 if (RT_SUCCESS(rc))
755 {
756 /*
757 * Insert it into the free list and process pending frees.
758 */
759 rtmemBlockDelayInsert(pBlock);
760 while ((pBlock = rtmemBlockDelayRemove()) != NULL)
761 {
762 pv = pBlock->Core.Key;
763# ifdef RTALLOC_EFENCE_IN_FRONT
764 void *pvBlock = (char *)pv - cbFence;
765# else
766 void *pvBlock = (void *)((uintptr_t)pv & ~RTSystemGetPageOffsetMask());
767# endif
768 size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, cbPage) + cbFence;
769 rc = RTMemProtect(pvBlock, cbBlock, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
770 if (RT_SUCCESS(rc))
771 RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, cbPage) + cbFence);
772 else
773 rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ | RTMEM_PROT_WRITE) -> %d\n", pvBlock, cbBlock, rc);
774 rtmemBlockFree(pBlock);
775 }
776 }
777 else
778 rtmemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc);
779
780# else /* !RTALLOC_EFENCE_FREE_DELAYED */
781
782 /*
783 * Turn of the E-fence and free it.
784 */
785# ifdef RTALLOC_EFENCE_IN_FRONT
786 void *pvBlock = (char *)pv - cbFence;
787 void *pvEFence = pvBlock;
788# else
789 void *pvBlock = (void *)((uintptr_t)pv & ~RTSystemGetPageOffsetMask());
790 void *pvEFence = (char *)pv + pBlock->cb;
791# endif
792 int rc = RTMemProtect(pvEFence, cbFence, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
793 if (RT_SUCCESS(rc))
794 RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, cbPage) + cbFence);
795 else
796 rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ | RTMEM_PROT_WRITE) -> %d\n", pvEFence, cbFence, rc);
797 rtmemBlockFree(pBlock);
798
799# endif /* !RTALLOC_EFENCE_FREE_DELAYED */
800 }
801 else
802 rtmemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv);
803
804#else /* !RTALLOC_EFENCE_TRACE */
805
806 /*
807 * We have no size tracking, so we're not doing any freeing because
808 * we cannot if the E-fence is after the block.
809 * Let's just expand the E-fence to the first page of the user bit
810 * since we know that it's around.
811 */
812 if (enmType == RTMEMTYPE_RTMEMFREEZ)
813 RT_BZERO(pv, cbUser);
814 int rc = RTMemProtect((void *)((uintptr_t)pv & ~RTSystemGetPageOffsetMask()), cbPage, RTMEM_PROT_NONE);
815 if (RT_FAILURE(rc))
816 rtmemComplain(pszOp, "RTMemProtect(%p, cbPage, RTMEM_PROT_NONE) -> %d\n", (void *)((uintptr_t)pv & ~RTSystemGetPageOffsetMask()), rc);
817#endif /* !RTALLOC_EFENCE_TRACE */
818}
819
820
821/**
822 * Internal realloc.
823 */
824RTDECL(void *) rtR3MemRealloc(const char *pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew,
825 const char *pszTag, void *pvCaller, RT_SRC_POS_DECL)
826{
827 /*
828 * Allocate new and copy.
829 */
830 if (!pvOld)
831 return rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
832 if (!cbNew)
833 {
834 rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, 0, pvCaller, RT_SRC_POS_ARGS);
835 return NULL;
836 }
837
838#ifdef RTALLOC_EFENCE_TRACE
839
840 /*
841 * Get the block, allocate the new, copy the data, free the old one.
842 */
843 PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
844 if (pBlock)
845 {
846 void *pvRet = rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
847 if (pvRet)
848 {
849 memcpy(pvRet, pvOld, RT_MIN(cbNew, pBlock->cbUnaligned));
850 rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, 0, pvCaller, RT_SRC_POS_ARGS);
851 }
852 return pvRet;
853 }
854 else
855 rtmemComplain(pszOp, "pvOld=%p was not found!\n", pvOld);
856 return NULL;
857
858#else /* !RTALLOC_EFENCE_TRACE */
859
860 rtmemComplain(pszOp, "Not supported if RTALLOC_EFENCE_TRACE isn't defined!\n");
861 return NULL;
862
863#endif /* !RTALLOC_EFENCE_TRACE */
864}
865
866
867
868
869RTDECL(void *) RTMemEfTmpAlloc(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
870{
871 return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
872}
873
874
875RTDECL(void *) RTMemEfTmpAllocZ(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
876{
877 return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
878}
879
880
881RTDECL(void) RTMemEfTmpFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
882{
883 if (pv)
884 rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS_ARGS);
885}
886
887
888RTDECL(void) RTMemEfTmpFreeZ(void *pv, size_t cb, RT_SRC_POS_DECL) RT_NO_THROW_DEF
889{
890 if (pv)
891 rtR3MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), RT_SRC_POS_ARGS);
892}
893
894
895RTDECL(void *) RTMemEfAlloc(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
896{
897 return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
898}
899
900
901RTDECL(void *) RTMemEfAllocZ(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
902{
903 return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
904}
905
906
907RTDECL(void *) RTMemEfAllocVar(size_t cbUnaligned, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
908{
909 size_t cbAligned;
910 if (cbUnaligned >= 16)
911 cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
912 else
913 cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
914 return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
915}
916
917
918RTDECL(void *) RTMemEfAllocZVar(size_t cbUnaligned, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
919{
920 size_t cbAligned;
921 if (cbUnaligned >= 16)
922 cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
923 else
924 cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
925 return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
926}
927
928
929RTDECL(void *) RTMemEfRealloc(void *pvOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
930{
931 return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
932}
933
934
935RTDECL(void *) RTMemEfReallocZ(void *pvOld, size_t cbOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
936{
937 void *pvDst = rtR3MemRealloc("ReallocZ", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
938 if (pvDst && cbNew > cbOld)
939 memset((uint8_t *)pvDst + cbOld, 0, cbNew - cbOld);
940 return pvDst;
941}
942
943
944RTDECL(void) RTMemEfFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
945{
946 if (pv)
947 rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS_ARGS);
948}
949
950
951RTDECL(void) RTMemEfFreeZ(void *pv, size_t cb, RT_SRC_POS_DECL) RT_NO_THROW_DEF
952{
953 if (pv)
954 rtR3MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), RT_SRC_POS_ARGS);
955}
956
957
958RTDECL(void *) RTMemEfDup(const void *pvSrc, size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
959{
960 void *pvDst = RTMemEfAlloc(cb, pszTag, RT_SRC_POS_ARGS);
961 if (pvDst)
962 memcpy(pvDst, pvSrc, cb);
963 return pvDst;
964}
965
966
967RTDECL(void *) RTMemEfDupEx(const void *pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
968{
969 void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, RT_SRC_POS_ARGS);
970 if (pvDst)
971 {
972 memcpy(pvDst, pvSrc, cbSrc);
973 memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
974 }
975 return pvDst;
976}
977
978
979
980
981/*
982 *
983 * The NP (no position) versions.
984 *
985 */
986
987
988
989RTDECL(void *) RTMemEfTmpAllocNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
990{
991 return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
992}
993
994
995RTDECL(void *) RTMemEfTmpAllocZNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
996{
997 return rtR3MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
998}
999
1000
1001RTDECL(void) RTMemEfTmpFreeNP(void *pv) RT_NO_THROW_DEF
1002{
1003 if (pv)
1004 rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), NULL, 0, NULL);
1005}
1006
1007
1008RTDECL(void) RTMemEfTmpFreeZNP(void *pv, size_t cb) RT_NO_THROW_DEF
1009{
1010 if (pv)
1011 rtR3MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), NULL, 0, NULL);
1012}
1013
1014
1015RTDECL(void *) RTMemEfAllocNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
1016{
1017 return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1018}
1019
1020
1021RTDECL(void *) RTMemEfAllocZNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF
1022{
1023 return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1024}
1025
1026
1027RTDECL(void *) RTMemEfAllocVarNP(size_t cbUnaligned, const char *pszTag) RT_NO_THROW_DEF
1028{
1029 size_t cbAligned;
1030 if (cbUnaligned >= 16)
1031 cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
1032 else
1033 cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
1034 return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1035}
1036
1037
1038RTDECL(void *) RTMemEfAllocZVarNP(size_t cbUnaligned, const char *pszTag) RT_NO_THROW_DEF
1039{
1040 size_t cbAligned;
1041 if (cbUnaligned >= 16)
1042 cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
1043 else
1044 cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
1045 return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1046}
1047
1048
1049RTDECL(void *) RTMemEfReallocNP(void *pvOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
1050{
1051 return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1052}
1053
1054
1055RTDECL(void *) RTMemEfReallocZNP(void *pvOld, size_t cbOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
1056{
1057 void *pvDst = rtR3MemRealloc("ReallocZ", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
1058 if (pvDst && cbNew > cbOld)
1059 memset((uint8_t *)pvDst + cbOld, 0, cbNew - cbOld);
1060 return pvDst;
1061}
1062
1063
1064RTDECL(void) RTMemEfFreeNP(void *pv) RT_NO_THROW_DEF
1065{
1066 if (pv)
1067 rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), NULL, 0, NULL);
1068}
1069
1070
1071RTDECL(void) RTMemEfFreeZNP(void *pv, size_t cb) RT_NO_THROW_DEF
1072{
1073 if (pv)
1074 rtR3MemFree("Free", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), NULL, 0, NULL);
1075}
1076
1077
1078RTDECL(void *) RTMemEfDupNP(const void *pvSrc, size_t cb, const char *pszTag) RT_NO_THROW_DEF
1079{
1080 void *pvDst = RTMemEfAlloc(cb, pszTag, NULL, 0, NULL);
1081 if (pvDst)
1082 memcpy(pvDst, pvSrc, cb);
1083 return pvDst;
1084}
1085
1086
1087RTDECL(void *) RTMemEfDupExNP(const void *pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag) RT_NO_THROW_DEF
1088{
1089 void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, NULL, 0, NULL);
1090 if (pvDst)
1091 {
1092 memcpy(pvDst, pvSrc, cbSrc);
1093 memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
1094 }
1095 return pvDst;
1096}
1097
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use