VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/darwin/memobj-r0drv-darwin.cpp@ 96407

Last change on this file since 96407 was 96407, checked in by vboxsync, 22 months ago

scm copyright and license note update
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 60.8 KB
Line 
1/* $Id: memobj-r0drv-darwin.cpp 96407 2022-08-22 17:43:14Z vboxsync $ */
2/** @file
3 * IPRT - Ring-0 Memory Objects, Darwin.
4 */
5
6/*
7 * Copyright (C) 2006-2022 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * The contents of this file may alternatively be used under the terms
26 * of the Common Development and Distribution License Version 1.0
27 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28 * in the VirtualBox distribution, in which case the provisions of the
29 * CDDL are applicable instead of those of the GPL.
30 *
31 * You may elect to license modified versions of this file under the
32 * terms and conditions of either the GPL or the CDDL or both.
33 *
34 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35 */
36
37
38/*********************************************************************************************************************************
39* Header Files *
40*********************************************************************************************************************************/
41#define RTMEM_NO_WRAP_TO_EF_APIS /* circular dependency otherwise. */
42#include "the-darwin-kernel.h"
43#include "internal/iprt.h"
44#include <iprt/memobj.h>
45
46#include <iprt/asm.h>
47#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
48# include <iprt/x86.h>
49# include <iprt/asm-amd64-x86.h>
50#endif
51#include <iprt/assert.h>
52#include <iprt/log.h>
53#include <iprt/mem.h>
54#include <iprt/param.h>
55#include <iprt/process.h>
56#include <iprt/semaphore.h>
57#include <iprt/string.h>
58#include <iprt/thread.h>
59#include "internal/memobj.h"
60
61
62/*********************************************************************************************************************************
63* Defined Constants And Macros *
64*********************************************************************************************************************************/
65#define MY_PRINTF(...) do { printf(__VA_ARGS__); kprintf(__VA_ARGS__); } while (0)
66
67/*#define USE_VM_MAP_WIRE - may re-enable later when non-mapped allocations are added. */
68
69
70/*********************************************************************************************************************************
71* Structures and Typedefs *
72*********************************************************************************************************************************/
73/**
74 * The Darwin version of the memory object structure.
75 */
76typedef struct RTR0MEMOBJDARWIN
77{
78 /** The core structure. */
79 RTR0MEMOBJINTERNAL Core;
80 /** Pointer to the memory descriptor created for allocated and locked memory. */
81 IOMemoryDescriptor *pMemDesc;
82 /** Pointer to the memory mapping object for mapped memory. */
83 IOMemoryMap *pMemMap;
84} RTR0MEMOBJDARWIN, *PRTR0MEMOBJDARWIN;
85
86/**
87 * Common thread_call_allocate/thread_call_enter argument package.
88 */
89typedef struct RTR0MEMOBJDARWINTHREADARGS
90{
91 int32_t volatile rc;
92 RTSEMEVENTMULTI hEvent;
93} RTR0MEMOBJDARWINTHREADARGS;
94
95
96/**
97 * Arguments for rtR0MemObjNativeAllockWorkOnKernelThread.
98 */
99typedef struct RTR0MEMOBJDARWINALLOCARGS
100{
101 RTR0MEMOBJDARWINTHREADARGS Core;
102 PPRTR0MEMOBJINTERNAL ppMem;
103 size_t cb;
104 bool fExecutable;
105 bool fContiguous;
106 mach_vm_address_t PhysMask;
107 uint64_t MaxPhysAddr;
108 RTR0MEMOBJTYPE enmType;
109 size_t uAlignment;
110 const char *pszTag;
111} RTR0MEMOBJDARWINALLOCARGS;
112
113/**
114 * Arguments for rtR0MemObjNativeProtectWorkOnKernelThread.
115 */
116typedef struct RTR0MEMOBJDARWINPROTECTARGS
117{
118 RTR0MEMOBJDARWINTHREADARGS Core;
119 PRTR0MEMOBJINTERNAL pMem;
120 size_t offSub;
121 size_t cbSub;
122 uint32_t fProt;
123} RTR0MEMOBJDARWINPROTECTARGS;
124
125
126/*********************************************************************************************************************************
127* Internal Functions *
128*********************************************************************************************************************************/
129static void rtR0MemObjNativeAllockWorkerOnKernelThread(void *pvUser0, void *pvUser1);
130static int rtR0MemObjNativeProtectWorker(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt);
131static void rtR0MemObjNativeProtectWorkerOnKernelThread(void *pvUser0, void *pvUser1);
132
133
134/**
135 * Touch the pages to force the kernel to create or write-enable the page table
136 * entries.
137 *
138 * This is necessary since the kernel gets upset if we take a page fault when
139 * preemption is disabled and/or we own a simple lock (same thing). It has no
140 * problems with us disabling interrupts when taking the traps, weird stuff.
141 *
142 * (This is basically a way of invoking vm_fault on a range of pages.)
143 *
144 * @param pv Pointer to the first page.
145 * @param cb The number of bytes.
146 */
147static void rtR0MemObjDarwinTouchPages(void *pv, size_t cb)
148{
149 uint32_t volatile *pu32 = (uint32_t volatile *)pv;
150 for (;;)
151 {
152 ASMAtomicCmpXchgU32(pu32, 0xdeadbeef, 0xdeadbeef);
153 if (cb <= PAGE_SIZE)
154 break;
155 cb -= PAGE_SIZE;
156 pu32 += PAGE_SIZE / sizeof(uint32_t);
157 }
158}
159
160
161/**
162 * Read (sniff) every page in the range to make sure there are some page tables
163 * entries backing it.
164 *
165 * This is just to be sure vm_protect didn't remove stuff without re-adding it
166 * if someone should try write-protect something.
167 *
168 * @param pv Pointer to the first page.
169 * @param cb The number of bytes.
170 */
171static void rtR0MemObjDarwinSniffPages(void const *pv, size_t cb)
172{
173 uint32_t volatile *pu32 = (uint32_t volatile *)pv;
174 uint32_t volatile u32Counter = 0;
175 for (;;)
176 {
177 u32Counter += *pu32;
178
179 if (cb <= PAGE_SIZE)
180 break;
181 cb -= PAGE_SIZE;
182 pu32 += PAGE_SIZE / sizeof(uint32_t);
183 }
184}
185
186
187/**
188 * Gets the virtual memory map the specified object is mapped into.
189 *
190 * @returns VM map handle on success, NULL if no map.
191 * @param pMem The memory object.
192 */
193DECLINLINE(vm_map_t) rtR0MemObjDarwinGetMap(PRTR0MEMOBJINTERNAL pMem)
194{
195 switch (pMem->enmType)
196 {
197 case RTR0MEMOBJTYPE_PAGE:
198 case RTR0MEMOBJTYPE_LOW:
199 case RTR0MEMOBJTYPE_CONT:
200 return kernel_map;
201
202 case RTR0MEMOBJTYPE_PHYS:
203 case RTR0MEMOBJTYPE_PHYS_NC:
204 if (pMem->pv)
205 return kernel_map;
206 return NULL;
207
208 case RTR0MEMOBJTYPE_LOCK:
209 return pMem->u.Lock.R0Process == NIL_RTR0PROCESS
210 ? kernel_map
211 : get_task_map((task_t)pMem->u.Lock.R0Process);
212
213 case RTR0MEMOBJTYPE_RES_VIRT:
214 return pMem->u.ResVirt.R0Process == NIL_RTR0PROCESS
215 ? kernel_map
216 : get_task_map((task_t)pMem->u.ResVirt.R0Process);
217
218 case RTR0MEMOBJTYPE_MAPPING:
219 return pMem->u.Mapping.R0Process == NIL_RTR0PROCESS
220 ? kernel_map
221 : get_task_map((task_t)pMem->u.Mapping.R0Process);
222
223 default:
224 return NULL;
225 }
226}
227
228#if 0 /* not necessary after all*/
229/* My vm_map mockup. */
230struct my_vm_map
231{
232 struct { char pad[8]; } lock;
233 struct my_vm_map_header
234 {
235 struct vm_map_links
236 {
237 void *prev;
238 void *next;
239 vm_map_offset_t start;
240 vm_map_offset_t end;
241 } links;
242 int nentries;
243 boolean_t entries_pageable;
244 } hdr;
245 pmap_t pmap;
246 vm_map_size_t size;
247};
248
249
250/**
251 * Gets the minimum map address, this is similar to get_map_min.
252 *
253 * @returns The start address of the map.
254 * @param pMap The map.
255 */
256static vm_map_offset_t rtR0MemObjDarwinGetMapMin(vm_map_t pMap)
257{
258 /* lazy discovery of the correct offset. The apple guys is a wonderfully secretive bunch. */
259 static int32_t volatile s_offAdjust = INT32_MAX;
260 int32_t off = s_offAdjust;
261 if (off == INT32_MAX)
262 {
263 for (off = 0; ; off += sizeof(pmap_t))
264 {
265 if (*(pmap_t *)((uint8_t *)kernel_map + off) == kernel_pmap)
266 break;
267 AssertReturn(off <= RT_MAX(RT_OFFSETOF(struct my_vm_map, pmap) * 4, 1024), 0x1000);
268 }
269 ASMAtomicWriteS32(&s_offAdjust, off - RT_OFFSETOF(struct my_vm_map, pmap));
270 }
271
272 /* calculate it. */
273 struct my_vm_map *pMyMap = (struct my_vm_map *)((uint8_t *)pMap + off);
274 return pMyMap->hdr.links.start;
275}
276#endif /* unused */
277
278#ifdef RT_STRICT
279# if 0 /* unused */
280
281/**
282 * Read from a physical page.
283 *
284 * @param HCPhys The address to start reading at.
285 * @param cb How many bytes to read.
286 * @param pvDst Where to put the bytes. This is zero'd on failure.
287 */
288static void rtR0MemObjDarwinReadPhys(RTHCPHYS HCPhys, size_t cb, void *pvDst)
289{
290 memset(pvDst, '\0', cb);
291
292 IOAddressRange aRanges[1] = { { (mach_vm_address_t)HCPhys, RT_ALIGN_Z(cb, PAGE_SIZE) } };
293 IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRanges(&aRanges[0], RT_ELEMENTS(aRanges),
294 kIODirectionIn, NULL /*task*/);
295 if (pMemDesc)
296 {
297#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
298 IOMemoryMap *pMemMap = pMemDesc->createMappingInTask(kernel_task, 0, kIOMapAnywhere | kIOMapDefaultCache);
299#else
300 IOMemoryMap *pMemMap = pMemDesc->map(kernel_task, 0, kIOMapAnywhere | kIOMapDefaultCache);
301#endif
302 if (pMemMap)
303 {
304 void const *pvSrc = (void const *)(uintptr_t)pMemMap->getVirtualAddress();
305 memcpy(pvDst, pvSrc, cb);
306 pMemMap->release();
307 }
308 else
309 MY_PRINTF("rtR0MemObjDarwinReadPhys: createMappingInTask failed; HCPhys=%llx\n", HCPhys);
310
311 pMemDesc->release();
312 }
313 else
314 MY_PRINTF("rtR0MemObjDarwinReadPhys: withAddressRanges failed; HCPhys=%llx\n", HCPhys);
315}
316
317
318/**
319 * Gets the PTE for a page.
320 *
321 * @returns the PTE.
322 * @param pvPage The virtual address to get the PTE for.
323 */
324static uint64_t rtR0MemObjDarwinGetPTE(void *pvPage)
325{
326 RTUINT64U u64;
327 RTCCUINTREG cr3 = ASMGetCR3();
328 RTCCUINTREG cr4 = ASMGetCR4();
329 bool fPAE = false;
330 bool fLMA = false;
331 if (cr4 & X86_CR4_PAE)
332 {
333 fPAE = true;
334 uint32_t fExtFeatures = ASMCpuId_EDX(0x80000001);
335 if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
336 {
337 uint64_t efer = ASMRdMsr(MSR_K6_EFER);
338 if (efer & MSR_K6_EFER_LMA)
339 fLMA = true;
340 }
341 }
342
343 if (fLMA)
344 {
345 /* PML4 */
346 rtR0MemObjDarwinReadPhys((cr3 & ~(RTCCUINTREG)PAGE_OFFSET_MASK) | (((uint64_t)(uintptr_t)pvPage >> X86_PML4_SHIFT) & X86_PML4_MASK) * 8, 8, &u64);
347 if (!(u64.u & X86_PML4E_P))
348 {
349 MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PML4E !p\n", pvPage);
350 return 0;
351 }
352
353 /* PDPTR */
354 rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64) * 8, 8, &u64);
355 if (!(u64.u & X86_PDPE_P))
356 {
357 MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PDPTE !p\n", pvPage);
358 return 0;
359 }
360 if (u64.u & X86_PDPE_LM_PS)
361 return (u64.u & ~(uint64_t)(_1G -1)) | ((uintptr_t)pvPage & (_1G -1));
362
363 /* PD */
364 rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK) * 8, 8, &u64);
365 if (!(u64.u & X86_PDE_P))
366 {
367 MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PDE !p\n", pvPage);
368 return 0;
369 }
370 if (u64.u & X86_PDE_PS)
371 return (u64.u & ~(uint64_t)(_2M -1)) | ((uintptr_t)pvPage & (_2M -1));
372
373 /* PT */
374 rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK) * 8, 8, &u64);
375 if (!(u64.u & X86_PTE_P))
376 {
377 MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PTE !p\n", pvPage);
378 return 0;
379 }
380 return u64.u;
381 }
382
383 if (fPAE)
384 {
385 /* PDPTR */
386 rtR0MemObjDarwinReadPhys((u64.u & X86_CR3_PAE_PAGE_MASK) | (((uintptr_t)pvPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE) * 8, 8, &u64);
387 if (!(u64.u & X86_PDE_P))
388 return 0;
389
390 /* PD */
391 rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK) * 8, 8, &u64);
392 if (!(u64.u & X86_PDE_P))
393 return 0;
394 if (u64.u & X86_PDE_PS)
395 return (u64.u & ~(uint64_t)(_2M -1)) | ((uintptr_t)pvPage & (_2M -1));
396
397 /* PT */
398 rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK) * 8, 8, &u64);
399 if (!(u64.u & X86_PTE_P))
400 return 0;
401 return u64.u;
402 }
403
404 /* PD */
405 rtR0MemObjDarwinReadPhys((u64.au32[0] & ~(uint32_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PD_SHIFT) & X86_PD_MASK) * 4, 4, &u64);
406 if (!(u64.au32[0] & X86_PDE_P))
407 return 0;
408 if (u64.au32[0] & X86_PDE_PS)
409 return (u64.u & ~(uint64_t)(_2M -1)) | ((uintptr_t)pvPage & (_2M -1));
410
411 /* PT */
412 rtR0MemObjDarwinReadPhys((u64.au32[0] & ~(uint32_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PT_SHIFT) & X86_PT_MASK) * 4, 4, &u64);
413 if (!(u64.au32[0] & X86_PTE_P))
414 return 0;
415 return u64.au32[0];
416
417 return 0;
418}
419
420# endif /* unused */
421#endif /* RT_STRICT */
422
423DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
424{
425 PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)pMem;
426 IPRT_DARWIN_SAVE_EFL_AC();
427
428 /*
429 * Release the IOMemoryDescriptor or/and IOMemoryMap associated with the object.
430 */
431 if (pMemDarwin->pMemDesc)
432 {
433 pMemDarwin->pMemDesc->complete();
434 pMemDarwin->pMemDesc->release();
435 pMemDarwin->pMemDesc = NULL;
436 }
437
438 if (pMemDarwin->pMemMap)
439 {
440 pMemDarwin->pMemMap->release();
441 pMemDarwin->pMemMap = NULL;
442 }
443
444 /*
445 * Release any memory that we've allocated or locked.
446 */
447 switch (pMemDarwin->Core.enmType)
448 {
449 case RTR0MEMOBJTYPE_LOW:
450 case RTR0MEMOBJTYPE_PAGE:
451 case RTR0MEMOBJTYPE_CONT:
452 break;
453
454 case RTR0MEMOBJTYPE_LOCK:
455 {
456#ifdef USE_VM_MAP_WIRE
457 vm_map_t Map = pMemDarwin->Core.u.Lock.R0Process != NIL_RTR0PROCESS
458 ? get_task_map((task_t)pMemDarwin->Core.u.Lock.R0Process)
459 : kernel_map;
460 kern_return_t kr = vm_map_unwire(Map,
461 (vm_map_offset_t)pMemDarwin->Core.pv,
462 (vm_map_offset_t)pMemDarwin->Core.pv + pMemDarwin->Core.cb,
463 0 /* not user */);
464 AssertRC(kr == KERN_SUCCESS); /** @todo don't ignore... */
465#endif
466 break;
467 }
468
469 case RTR0MEMOBJTYPE_PHYS:
470 /*if (pMemDarwin->Core.u.Phys.fAllocated)
471 IOFreePhysical(pMemDarwin->Core.u.Phys.PhysBase, pMemDarwin->Core.cb);*/
472 Assert(!pMemDarwin->Core.u.Phys.fAllocated);
473 break;
474
475 case RTR0MEMOBJTYPE_PHYS_NC:
476 AssertMsgFailed(("RTR0MEMOBJTYPE_PHYS_NC\n"));
477 IPRT_DARWIN_RESTORE_EFL_AC();
478 return VERR_INTERNAL_ERROR;
479
480 case RTR0MEMOBJTYPE_RES_VIRT:
481 AssertMsgFailed(("RTR0MEMOBJTYPE_RES_VIRT\n"));
482 IPRT_DARWIN_RESTORE_EFL_AC();
483 return VERR_INTERNAL_ERROR;
484
485 case RTR0MEMOBJTYPE_MAPPING:
486 /* nothing to do here. */
487 break;
488
489 default:
490 AssertMsgFailed(("enmType=%d\n", pMemDarwin->Core.enmType));
491 IPRT_DARWIN_RESTORE_EFL_AC();
492 return VERR_INTERNAL_ERROR;
493 }
494
495 IPRT_DARWIN_RESTORE_EFL_AC();
496 return VINF_SUCCESS;
497}
498
499
500/**
501 * This is a helper function to executes @a pfnWorker in the context of the
502 * kernel_task
503 *
504 * @returns IPRT status code - result from pfnWorker or dispatching error.
505 * @param pfnWorker The function to call.
506 * @param pArgs The arguments to pass to the function.
507 */
508static int rtR0MemObjDarwinDoInKernelTaskThread(thread_call_func_t pfnWorker, RTR0MEMOBJDARWINTHREADARGS *pArgs)
509{
510 pArgs->rc = VERR_IPE_UNINITIALIZED_STATUS;
511 pArgs->hEvent = NIL_RTSEMEVENTMULTI;
512 int rc = RTSemEventMultiCreate(&pArgs->hEvent);
513 if (RT_SUCCESS(rc))
514 {
515 thread_call_t hCall = thread_call_allocate(pfnWorker, (void *)pArgs);
516 if (hCall)
517 {
518 boolean_t fRc = thread_call_enter(hCall);
519 AssertLogRel(fRc == FALSE);
520
521 rc = RTSemEventMultiWaitEx(pArgs->hEvent, RTSEMWAIT_FLAGS_INDEFINITE | RTSEMWAIT_FLAGS_UNINTERRUPTIBLE,
522 RT_INDEFINITE_WAIT);
523 AssertLogRelRC(rc);
524
525 rc = pArgs->rc;
526 thread_call_free(hCall);
527 }
528 else
529 rc = VERR_NO_MEMORY;
530 RTSemEventMultiDestroy(pArgs->hEvent);
531 }
532 return rc;
533}
534
535
536/**
537 * Signals result to thread waiting in rtR0MemObjDarwinDoInKernelTaskThread.
538 *
539 * @param pArgs The argument structure.
540 * @param rc The IPRT status code to signal.
541 */
542static void rtR0MemObjDarwinSignalThreadWaitinOnTask(RTR0MEMOBJDARWINTHREADARGS volatile *pArgs, int rc)
543{
544 if (ASMAtomicCmpXchgS32(&pArgs->rc, rc, VERR_IPE_UNINITIALIZED_STATUS))
545 {
546 rc = RTSemEventMultiSignal(pArgs->hEvent);
547 AssertLogRelRC(rc);
548 }
549}
550
551
552/**
553 * Kernel memory alloc worker that uses inTaskWithPhysicalMask.
554 *
555 * @returns IPRT status code.
556 * @retval VERR_ADDRESS_TOO_BIG try another way.
557 *
558 * @param ppMem Where to return the memory object.
559 * @param cb The page aligned memory size.
560 * @param fExecutable Whether the mapping needs to be executable.
561 * @param fContiguous Whether the backing memory needs to be contiguous.
562 * @param PhysMask The mask for the backing memory (i.e. range). Use 0 if
563 * you don't care that much or is speculating.
564 * @param MaxPhysAddr The max address to verify the result against. Use
565 * UINT64_MAX if it doesn't matter.
566 * @param enmType The object type.
567 * @param uAlignment The allocation alignment (in bytes).
568 * @param pszTag Allocation tag used for statistics and such.
569 * @param fOnKernelThread Set if we're already on the kernel thread.
570 */
571static int rtR0MemObjNativeAllocWorker(PPRTR0MEMOBJINTERNAL ppMem, size_t cb,
572 bool fExecutable, bool fContiguous,
573 mach_vm_address_t PhysMask, uint64_t MaxPhysAddr,
574 RTR0MEMOBJTYPE enmType, size_t uAlignment, const char *pszTag, bool fOnKernelThread)
575{
576 int rc;
577
578 /*
579 * Because of process code signing properties leaking into kernel space in
580 * in XNU's vm_fault.c code, we have to defer allocations of exec memory to
581 * a thread running in the kernel_task to get consistent results here.
582 *
583 * Trouble strikes in vm_fault_enter() when cs_enforcement_enabled is determined
584 * to be true because current process has the CS_ENFORCEMENT flag, the page flag
585 * vmp_cs_validated is clear, and the protection mask includes VM_PROT_EXECUTE
586 * (pmap_cs_enforced does not apply to macOS it seems). This test seems to go
587 * back to 10.5, though I'm not sure whether it's enabled for macOS that early
588 * on. Only VM_PROT_EXECUTE is problematic for kernel memory, (though
589 * VM_PROT_WRITE on code signed pages is also problematic in theory). As long as
590 * kernel_task doesn't have CS_ENFORCEMENT enabled, we'll be fine switching to it.
591 */
592 if (!fExecutable || fOnKernelThread)
593 { /* likely */ }
594 else
595 {
596 RTR0MEMOBJDARWINALLOCARGS Args;
597 Args.ppMem = ppMem;
598 Args.cb = cb;
599 Args.fExecutable = fExecutable;
600 Args.fContiguous = fContiguous;
601 Args.PhysMask = PhysMask;
602 Args.MaxPhysAddr = MaxPhysAddr;
603 Args.enmType = enmType;
604 Args.uAlignment = uAlignment;
605 Args.pszTag = pszTag;
606 return rtR0MemObjDarwinDoInKernelTaskThread(rtR0MemObjNativeAllockWorkerOnKernelThread, &Args.Core);
607 }
608
609 /*
610 * Try inTaskWithPhysicalMask first, but since we don't quite trust that it
611 * actually respects the physical memory mask (10.5.x is certainly busted),
612 * we'll use rtR0MemObjNativeAllocCont as a fallback for dealing with that.
613 *
614 * The kIOMemoryKernelUserShared flag just forces the result to be page aligned.
615 *
616 * The kIOMemoryMapperNone flag is required since 10.8.2 (IOMMU changes?).
617 */
618
619 /* This is an old fudge from the snow leoard days: "Is it only on snow leopard?
620 Seen allocating memory for the VM structure, last page corrupted or
621 inaccessible." Made it only apply to snow leopard and older for now. */
622 size_t cbFudged = cb;
623 if (version_major >= 11 /* 10 = 10.7.x = Lion. */)
624 { /* likely */ }
625 else
626 cbFudged += PAGE_SIZE;
627
628 IOOptionBits fOptions = kIOMemoryKernelUserShared | kIODirectionInOut;
629 if (fContiguous)
630 {
631 fOptions |= kIOMemoryPhysicallyContiguous;
632 if ( version_major > 12
633 || (version_major == 12 && version_minor >= 2) /* 10.8.2 = Mountain Kitten */ )
634 fOptions |= kIOMemoryHostPhysicallyContiguous; /* (Just to make ourselves clear, in case the xnu code changes.) */
635 }
636 if (version_major >= 12 /* 12 = 10.8.x = Mountain Kitten */)
637 fOptions |= kIOMemoryMapperNone;
638
639#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070 && 0 /* enable when/if necessary */
640 /* Paranoia: Don't misrepresent our intentions, we won't map kernel executable memory into ring-0. */
641 if (fExecutable && version_major >= 11 /* 10.7.x = Lion, as below */)
642 {
643 fOptions &= ~kIOMemoryKernelUserShared;
644 if (uAlignment < PAGE_SIZE)
645 uAlignment = PAGE_SIZE;
646 }
647#endif
648
649 /* The public initWithPhysicalMask virtual method appeared in 10.7.0, in
650 versions 10.5.0 up to 10.7.0 it was private, and 10.4.8-10.5.0 it was
651 x86 only and didn't have the alignment parameter (slot was different too). */
652 uint64_t uAlignmentActual = uAlignment;
653 IOBufferMemoryDescriptor *pMemDesc;
654#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070
655 if (version_major >= 11 /* 11 = 10.7.x = Lion, could probably allow 10.5.0+ here if we really wanted to. */)
656 {
657 /* Starting with 10.6.x the physical mask is ignored if alignment is higher
658 than 1. The assumption seems to be that inTaskWithPhysicalMask() should
659 be used and the alignment inferred from the PhysMask argument. */
660 if (MaxPhysAddr != UINT64_MAX)
661 {
662 Assert(RT_ALIGN_64(PhysMask, uAlignment) == PhysMask);
663 uAlignmentActual = 1;
664 }
665
666 pMemDesc = new IOBufferMemoryDescriptor;
667 if (pMemDesc)
668 {
669 if (pMemDesc->initWithPhysicalMask(kernel_task, fOptions, cbFudged, uAlignmentActual, PhysMask))
670 { /* likely */ }
671 else
672 {
673 pMemDesc->release();
674 pMemDesc = NULL;
675 }
676 }
677 }
678 else
679#endif
680 pMemDesc = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task, fOptions, cbFudged, PhysMask);
681 if (pMemDesc)
682 {
683 IOReturn IORet = pMemDesc->prepare(kIODirectionInOut);
684 if (IORet == kIOReturnSuccess)
685 {
686 void *pv = pMemDesc->getBytesNoCopy(0, cbFudged);
687 if (pv)
688 {
689 /*
690 * Check if it's all below 4GB.
691 */
692 addr64_t AddrPrev = 0;
693 MaxPhysAddr &= ~(uint64_t)PAGE_OFFSET_MASK;
694 for (IOByteCount off = 0; off < cb; off += PAGE_SIZE)
695 {
696#ifdef __LP64__
697 addr64_t Addr = pMemDesc->getPhysicalSegment(off, NULL, kIOMemoryMapperNone);
698#else
699 addr64_t Addr = pMemDesc->getPhysicalSegment64(off, NULL);
700#endif
701 if ( Addr > MaxPhysAddr
702 || !Addr
703 || (Addr & PAGE_OFFSET_MASK)
704 || ( fContiguous
705 && !off
706 && Addr == AddrPrev + PAGE_SIZE))
707 {
708 /* Buggy API, try allocate the memory another way. */
709 pMemDesc->complete();
710 pMemDesc->release();
711 if (PhysMask)
712 {
713 kprintf("rtR0MemObjNativeAllocWorker: off=%zx Addr=%llx AddrPrev=%llx MaxPhysAddr=%llx PhysMas=%llx fContiguous=%d fOptions=%#x - buggy API!\n",
714 (size_t)off, Addr, AddrPrev, MaxPhysAddr, PhysMask, fContiguous, fOptions);
715 LogRel(("rtR0MemObjNativeAllocWorker: off=%zx Addr=%llx AddrPrev=%llx MaxPhysAddr=%llx PhysMas=%llx fContiguous=%RTbool fOptions=%#x - buggy API!\n",
716 (size_t)off, Addr, AddrPrev, MaxPhysAddr, PhysMask, fContiguous, fOptions));
717 }
718 return VERR_ADDRESS_TOO_BIG;
719 }
720 AddrPrev = Addr;
721 }
722
723 /*
724 * Check that it's aligned correctly.
725 */
726 if ((uintptr_t)pv & (uAlignment - 1))
727 {
728 pMemDesc->complete();
729 pMemDesc->release();
730 if (PhysMask)
731 {
732 kprintf("rtR0MemObjNativeAllocWorker: pv=%p uAlignment=%#zx (MaxPhysAddr=%llx PhysMas=%llx fContiguous=%d fOptions=%#x) - buggy API!!\n",
733 pv, uAlignment, MaxPhysAddr, PhysMask, fContiguous, fOptions);
734 LogRel(("rtR0MemObjNativeAllocWorker: pv=%p uAlignment=%#zx (MaxPhysAddr=%llx PhysMas=%llx fContiguous=%RTbool fOptions=%#x) - buggy API!\n",
735 pv, uAlignment, MaxPhysAddr, PhysMask, fContiguous, fOptions));
736 }
737 return VERR_NOT_SUPPORTED;
738 }
739
740#ifdef RT_STRICT
741 /* check that the memory is actually mapped. */
742 //addr64_t Addr = pMemDesc->getPhysicalSegment64(0, NULL);
743 //printf("rtR0MemObjNativeAllocWorker: pv=%p %8llx %8llx\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr);
744 RTTHREADPREEMPTSTATE State = RTTHREADPREEMPTSTATE_INITIALIZER;
745 RTThreadPreemptDisable(&State);
746 rtR0MemObjDarwinTouchPages(pv, cb);
747 RTThreadPreemptRestore(&State);
748#endif
749
750 /*
751 * Create the IPRT memory object.
752 */
753 PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), enmType, pv, cb, pszTag);
754 if (pMemDarwin)
755 {
756 if (fOptions & kIOMemoryKernelUserShared)
757 pMemDarwin->Core.fFlags |= RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC;
758 else
759 pMemDarwin->Core.fFlags |= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC;
760 if (fContiguous)
761 {
762#ifdef __LP64__
763 addr64_t PhysBase64 = pMemDesc->getPhysicalSegment(0, NULL, kIOMemoryMapperNone);
764#else
765 addr64_t PhysBase64 = pMemDesc->getPhysicalSegment64(0, NULL);
766#endif
767 RTHCPHYS PhysBase = PhysBase64; Assert(PhysBase == PhysBase64);
768 if (enmType == RTR0MEMOBJTYPE_CONT)
769 pMemDarwin->Core.u.Cont.Phys = PhysBase;
770 else if (enmType == RTR0MEMOBJTYPE_PHYS)
771 pMemDarwin->Core.u.Phys.PhysBase = PhysBase;
772 else
773 AssertMsgFailed(("enmType=%d\n", enmType));
774 }
775
776 if (fExecutable)
777 {
778 rc = rtR0MemObjNativeProtectWorker(&pMemDarwin->Core, 0, cb,
779 RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC);
780#ifdef RT_STRICT
781 if (RT_SUCCESS(rc))
782 {
783 /* check that the memory is actually mapped. */
784 RTTHREADPREEMPTSTATE State2 = RTTHREADPREEMPTSTATE_INITIALIZER;
785 RTThreadPreemptDisable(&State2);
786 rtR0MemObjDarwinTouchPages(pv, cb);
787 RTThreadPreemptRestore(&State2);
788 }
789#endif
790 /* Bug 6226: Ignore KERN_PROTECTION_FAILURE on Leopard and older. */
791 if ( rc == VERR_PERMISSION_DENIED
792 && version_major <= 10 /* 10 = 10.6.x = Snow Leopard. */)
793 rc = VINF_SUCCESS;
794 }
795 else
796 rc = VINF_SUCCESS;
797 if (RT_SUCCESS(rc))
798 {
799 pMemDarwin->pMemDesc = pMemDesc;
800 *ppMem = &pMemDarwin->Core;
801 return VINF_SUCCESS;
802 }
803
804 rtR0MemObjDelete(&pMemDarwin->Core);
805 }
806
807 if (enmType == RTR0MEMOBJTYPE_PHYS_NC)
808 rc = VERR_NO_PHYS_MEMORY;
809 else if (enmType == RTR0MEMOBJTYPE_LOW)
810 rc = VERR_NO_LOW_MEMORY;
811 else if (enmType == RTR0MEMOBJTYPE_CONT)
812 rc = VERR_NO_CONT_MEMORY;
813 else
814 rc = VERR_NO_MEMORY;
815 }
816 else
817 rc = VERR_MEMOBJ_INIT_FAILED;
818
819 pMemDesc->complete();
820 }
821 else
822 rc = RTErrConvertFromDarwinIO(IORet);
823 pMemDesc->release();
824 }
825 else
826 rc = VERR_MEMOBJ_INIT_FAILED;
827 Assert(rc != VERR_ADDRESS_TOO_BIG);
828 return rc;
829}
830
831
832/**
833 * rtR0MemObjNativeAllocWorker kernel_task wrapper function.
834 */
835static void rtR0MemObjNativeAllockWorkerOnKernelThread(void *pvUser0, void *pvUser1)
836{
837 AssertPtr(pvUser0); Assert(pvUser1 == NULL); NOREF(pvUser1);
838 RTR0MEMOBJDARWINALLOCARGS volatile *pArgs = (RTR0MEMOBJDARWINALLOCARGS volatile *)pvUser0;
839 int rc = rtR0MemObjNativeAllocWorker(pArgs->ppMem, pArgs->cb, pArgs->fExecutable, pArgs->fContiguous, pArgs->PhysMask,
840 pArgs->MaxPhysAddr, pArgs->enmType, pArgs->uAlignment, pArgs->pszTag,
841 true /*fOnKernelThread*/);
842 rtR0MemObjDarwinSignalThreadWaitinOnTask(&pArgs->Core, rc);
843}
844
845
846DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
847{
848 IPRT_DARWIN_SAVE_EFL_AC();
849
850 int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, 0 /* PhysMask */, UINT64_MAX,
851 RTR0MEMOBJTYPE_PAGE, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);
852
853 IPRT_DARWIN_RESTORE_EFL_AC();
854 return rc;
855}
856
857
858DECLHIDDEN(int) rtR0MemObjNativeAllocLarge(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, size_t cbLargePage, uint32_t fFlags,
859 const char *pszTag)
860{
861 return rtR0MemObjFallbackAllocLarge(ppMem, cb, cbLargePage, fFlags, pszTag);
862}
863
864
865DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
866{
867 IPRT_DARWIN_SAVE_EFL_AC();
868
869 /*
870 * Try IOMallocPhysical/IOMallocAligned first.
871 * Then try optimistically without a physical address mask, which will always
872 * end up using IOMallocAligned.
873 *
874 * (See bug comment in the worker and IOBufferMemoryDescriptor::initWithPhysicalMask.)
875 */
876 int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, ~(uint32_t)PAGE_OFFSET_MASK,
877 _4G - PAGE_SIZE, RTR0MEMOBJTYPE_LOW, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);
878 if (rc == VERR_ADDRESS_TOO_BIG)
879 rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, 0 /* PhysMask */,
880 _4G - PAGE_SIZE, RTR0MEMOBJTYPE_LOW, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);
881
882 IPRT_DARWIN_RESTORE_EFL_AC();
883 return rc;
884}
885
886
887DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
888{
889 IPRT_DARWIN_SAVE_EFL_AC();
890
891 int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, true /* fContiguous */,
892 ~(uint32_t)PAGE_OFFSET_MASK, _4G - PAGE_SIZE,
893 RTR0MEMOBJTYPE_CONT, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);
894
895 /*
896 * Workaround for bogus IOKernelAllocateContiguous behavior, just in case.
897 * cb <= PAGE_SIZE allocations take a different path, using a different allocator.
898 */
899 if (RT_FAILURE(rc) && cb <= PAGE_SIZE)
900 rc = rtR0MemObjNativeAllocWorker(ppMem, cb + PAGE_SIZE, fExecutable, true /* fContiguous */,
901 ~(uint32_t)PAGE_OFFSET_MASK, _4G - PAGE_SIZE,
902 RTR0MEMOBJTYPE_CONT, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);
903 IPRT_DARWIN_RESTORE_EFL_AC();
904 return rc;
905}
906
907
908DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment,
909 const char *pszTag)
910{
911 if (uAlignment != PAGE_SIZE)
912 {
913 /* See rtR0MemObjNativeAllocWorker: */
914 if (version_major < 9 /* 9 = 10.5.x = Snow Leopard */)
915 return VERR_NOT_SUPPORTED;
916 }
917
918 IPRT_DARWIN_SAVE_EFL_AC();
919
920 /*
921 * Translate the PhysHighest address into a mask.
922 */
923 int rc;
924 if (PhysHighest == NIL_RTHCPHYS)
925 rc = rtR0MemObjNativeAllocWorker(ppMem, cb, false /* fExecutable */, true /* fContiguous */,
926 uAlignment <= PAGE_SIZE ? 0 : ~(mach_vm_address_t)(uAlignment - 1) /* PhysMask*/,
927 UINT64_MAX, RTR0MEMOBJTYPE_PHYS, uAlignment, pszTag, false /*fOnKernelThread*/);
928 else
929 {
930 mach_vm_address_t PhysMask = 0;
931 PhysMask = ~(mach_vm_address_t)0;
932 while (PhysMask > (PhysHighest | PAGE_OFFSET_MASK))
933 PhysMask >>= 1;
934 AssertReturn(PhysMask + 1 <= cb, VERR_INVALID_PARAMETER);
935 PhysMask &= ~(mach_vm_address_t)(uAlignment - 1);
936
937 rc = rtR0MemObjNativeAllocWorker(ppMem, cb, false /* fExecutable */, true /* fContiguous */,
938 PhysMask, PhysHighest,
939 RTR0MEMOBJTYPE_PHYS, uAlignment, pszTag, false /*fOnKernelThread*/);
940 }
941
942 IPRT_DARWIN_RESTORE_EFL_AC();
943 return rc;
944}
945
946
947DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, const char *pszTag)
948{
949 /** @todo rtR0MemObjNativeAllocPhys / darwin.
950 * This might be a bit problematic and may very well require having to create our own
951 * object which we populate with pages but without mapping it into any address space.
952 * Estimate is 2-3 days.
953 */
954 RT_NOREF(ppMem, cb, PhysHighest, pszTag);
955 return VERR_NOT_SUPPORTED;
956}
957
958
959DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy,
960 const char *pszTag)
961{
962 AssertReturn(uCachePolicy == RTMEM_CACHE_POLICY_DONT_CARE, VERR_NOT_SUPPORTED);
963 IPRT_DARWIN_SAVE_EFL_AC();
964
965 /*
966 * Create a descriptor for it (the validation is always true on intel macs, but
967 * as it doesn't harm us keep it in).
968 */
969 int rc = VERR_ADDRESS_TOO_BIG;
970 IOAddressRange aRanges[1] = { { Phys, cb } };
971 if ( aRanges[0].address == Phys
972 && aRanges[0].length == cb)
973 {
974 IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRanges(&aRanges[0], RT_ELEMENTS(aRanges),
975 kIODirectionInOut, NULL /*task*/);
976 if (pMemDesc)
977 {
978#ifdef __LP64__
979 Assert(Phys == pMemDesc->getPhysicalSegment(0, NULL, kIOMemoryMapperNone));
980#else
981 Assert(Phys == pMemDesc->getPhysicalSegment64(0, NULL));
982#endif
983
984 /*
985 * Create the IPRT memory object.
986 */
987 PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_PHYS,
988 NULL, cb, pszTag);
989 if (pMemDarwin)
990 {
991 pMemDarwin->Core.u.Phys.PhysBase = Phys;
992 pMemDarwin->Core.u.Phys.fAllocated = false;
993 pMemDarwin->Core.u.Phys.uCachePolicy = uCachePolicy;
994 pMemDarwin->pMemDesc = pMemDesc;
995 *ppMem = &pMemDarwin->Core;
996 IPRT_DARWIN_RESTORE_EFL_AC();
997 return VINF_SUCCESS;
998 }
999
1000 rc = VERR_NO_MEMORY;
1001 pMemDesc->release();
1002 }
1003 else
1004 rc = VERR_MEMOBJ_INIT_FAILED;
1005 }
1006 else
1007 AssertMsgFailed(("%#llx %llx\n", (unsigned long long)Phys, (unsigned long long)cb));
1008 IPRT_DARWIN_RESTORE_EFL_AC();
1009 return rc;
1010}
1011
1012
1013/**
1014 * Internal worker for locking down pages.
1015 *
1016 * @return IPRT status code.
1017 *
1018 * @param ppMem Where to store the memory object pointer.
1019 * @param pv First page.
1020 * @param cb Number of bytes.
1021 * @param fAccess The desired access, a combination of RTMEM_PROT_READ
1022 * and RTMEM_PROT_WRITE.
1023 * @param Task The task \a pv and \a cb refers to.
1024 * @param pszTag Allocation tag used for statistics and such.
1025 */
1026static int rtR0MemObjNativeLock(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess, task_t Task,
1027 const char *pszTag)
1028{
1029 IPRT_DARWIN_SAVE_EFL_AC();
1030 NOREF(fAccess);
1031#ifdef USE_VM_MAP_WIRE
1032 vm_map_t Map = get_task_map(Task);
1033 Assert(Map);
1034
1035 /*
1036 * First try lock the memory.
1037 */
1038 int rc = VERR_LOCK_FAILED;
1039 kern_return_t kr = vm_map_wire(get_task_map(Task),
1040 (vm_map_offset_t)pv,
1041 (vm_map_offset_t)pv + cb,
1042 VM_PROT_DEFAULT,
1043 0 /* not user */);
1044 if (kr == KERN_SUCCESS)
1045 {
1046 /*
1047 * Create the IPRT memory object.
1048 */
1049 PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_LOCK, pv, cb, pszTag);
1050 if (pMemDarwin)
1051 {
1052 pMemDarwin->Core.u.Lock.R0Process = (RTR0PROCESS)Task;
1053 *ppMem = &pMemDarwin->Core;
1054
1055 IPRT_DARWIN_RESTORE_EFL_AC();
1056 return VINF_SUCCESS;
1057 }
1058
1059 kr = vm_map_unwire(get_task_map(Task), (vm_map_offset_t)pv, (vm_map_offset_t)pv + cb, 0 /* not user */);
1060 Assert(kr == KERN_SUCCESS);
1061 rc = VERR_NO_MEMORY;
1062 }
1063
1064#else
1065
1066 /*
1067 * Create a descriptor and try lock it (prepare).
1068 */
1069 int rc = VERR_MEMOBJ_INIT_FAILED;
1070 IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRange((vm_address_t)pv, cb, kIODirectionInOut, Task);
1071 if (pMemDesc)
1072 {
1073 IOReturn IORet = pMemDesc->prepare(kIODirectionInOut);
1074 if (IORet == kIOReturnSuccess)
1075 {
1076 /*
1077 * Create the IPRT memory object.
1078 */
1079 PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_LOCK,
1080 pv, cb, pszTag);
1081 if (pMemDarwin)
1082 {
1083 pMemDarwin->Core.u.Lock.R0Process = (RTR0PROCESS)Task;
1084 pMemDarwin->pMemDesc = pMemDesc;
1085 *ppMem = &pMemDarwin->Core;
1086
1087 IPRT_DARWIN_RESTORE_EFL_AC();
1088 return VINF_SUCCESS;
1089 }
1090
1091 pMemDesc->complete();
1092 rc = VERR_NO_MEMORY;
1093 }
1094 else
1095 rc = VERR_LOCK_FAILED;
1096 pMemDesc->release();
1097 }
1098#endif
1099 IPRT_DARWIN_RESTORE_EFL_AC();
1100 return rc;
1101}
1102
1103
1104DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess,
1105 RTR0PROCESS R0Process, const char *pszTag)
1106{
1107 return rtR0MemObjNativeLock(ppMem, (void *)R3Ptr, cb, fAccess, (task_t)R0Process, pszTag);
1108}
1109
1110
1111DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess, const char *pszTag)
1112{
1113 return rtR0MemObjNativeLock(ppMem, pv, cb, fAccess, kernel_task, pszTag);
1114}
1115
1116
1117DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment,
1118 const char *pszTag)
1119{
1120 RT_NOREF(ppMem, pvFixed, cb, uAlignment, pszTag);
1121 return VERR_NOT_SUPPORTED;
1122}
1123
1124
1125DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment,
1126 RTR0PROCESS R0Process, const char *pszTag)
1127{
1128 RT_NOREF(ppMem, R3PtrFixed, cb, uAlignment, R0Process, pszTag);
1129 return VERR_NOT_SUPPORTED;
1130}
1131
1132
1133DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment,
1134 unsigned fProt, size_t offSub, size_t cbSub, const char *pszTag)
1135{
1136 RT_NOREF(fProt);
1137 AssertReturn(pvFixed == (void *)-1, VERR_NOT_SUPPORTED);
1138
1139 /*
1140 * Check that the specified alignment is supported.
1141 */
1142 if (uAlignment > PAGE_SIZE)
1143 return VERR_NOT_SUPPORTED;
1144 Assert(!offSub || cbSub);
1145
1146 IPRT_DARWIN_SAVE_EFL_AC();
1147
1148 /*
1149 * Must have a memory descriptor that we can map.
1150 */
1151 int rc = VERR_INVALID_PARAMETER;
1152 PRTR0MEMOBJDARWIN pMemToMapDarwin = (PRTR0MEMOBJDARWIN)pMemToMap;
1153 if (pMemToMapDarwin->pMemDesc)
1154 {
1155 /* The kIOMapPrefault option was added in 10.10.0; causes PTEs to be populated with
1156 INTEL_PTE_WIRED to be set, just like we desire (see further down). However, till
1157 10.13.0 it was not available for use on kernel mappings. Oh, fudge. */
1158#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
1159 static uint32_t volatile s_fOptions = UINT32_MAX;
1160 uint32_t fOptions = s_fOptions;
1161 if (RT_UNLIKELY(fOptions == UINT32_MAX))
1162 s_fOptions = fOptions = version_major >= 17 ? 0x10000000 /*kIOMapPrefault*/ : 0; /* Since 10.13.0 (High Sierra). */
1163
1164 IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask(kernel_task,
1165 0,
1166 kIOMapAnywhere | kIOMapDefaultCache | fOptions,
1167 offSub,
1168 cbSub);
1169#else
1170 IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->map(kernel_task,
1171 0,
1172 kIOMapAnywhere | kIOMapDefaultCache,
1173 offSub,
1174 cbSub);
1175#endif
1176 if (pMemMap)
1177 {
1178 IOVirtualAddress VirtAddr = pMemMap->getVirtualAddress();
1179 void *pv = (void *)(uintptr_t)VirtAddr;
1180 if ((uintptr_t)pv == VirtAddr && pv != NULL)
1181 {
1182//#ifdef __LP64__
1183// addr64_t Addr = pMemToMapDarwin->pMemDesc->getPhysicalSegment(offSub, NULL, kIOMemoryMapperNone);
1184//#else
1185// addr64_t Addr = pMemToMapDarwin->pMemDesc->getPhysicalSegment64(offSub, NULL);
1186//#endif
1187// MY_PRINTF("pv=%p: %8llx %8llx\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr);
1188
1189// /*
1190// * Explicitly lock it so that we're sure it is present and that
1191// * its PTEs cannot be recycled.
1192// * Note! withAddressRange() doesn't work as it adds kIOMemoryTypeVirtual64
1193// * to the options which causes prepare() to not wire the pages.
1194// * This is probably a bug.
1195// */
1196// IOAddressRange Range = { (mach_vm_address_t)pv, cbSub };
1197// IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withOptions(&Range,
1198// 1 /* count */,
1199// 0 /* offset */,
1200// kernel_task,
1201// kIODirectionInOut | kIOMemoryTypeVirtual,
1202// kIOMapperSystem);
1203// if (pMemDesc)
1204// {
1205// IOReturn IORet = pMemDesc->prepare(kIODirectionInOut);
1206// if (IORet == kIOReturnSuccess)
1207// {
1208 /* HACK ALERT! On kernels older than 10.10 (xnu version 14), we need to fault in
1209 the pages here so they can safely be accessed from inside simple
1210 locks and when preemption is disabled (no page-ins allowed).
1211 Note! This touching does not cause INTEL_PTE_WIRED (bit 10) to be set as we go
1212 thru general #PF and vm_fault doesn't figure it should be wired or something. */
1213 rtR0MemObjDarwinTouchPages(pv, cbSub ? cbSub : pMemToMap->cb);
1214 /** @todo First, the memory should've been mapped by now, and second, it
1215 * should have the wired attribute in the PTE (bit 10). Neither seems to
1216 * be the case. The disabled locking code doesn't make any difference,
1217 * which is extremely odd, and breaks rtR0MemObjNativeGetPagePhysAddr
1218 * (getPhysicalSegment64 -> 64 for the lock descriptor. */
1219//#ifdef __LP64__
1220// addr64_t Addr2 = pMemToMapDarwin->pMemDesc->getPhysicalSegment(offSub, NULL, kIOMemoryMapperNone);
1221//#else
1222// addr64_t Addr2 = pMemToMapDarwin->pMemDesc->getPhysicalSegment64(offSub, NULL);
1223//#endif
1224// MY_PRINTF("pv=%p: %8llx %8llx (%d)\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr2, 2);
1225
1226 /*
1227 * Create the IPRT memory object.
1228 */
1229 PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_MAPPING,
1230 pv, cbSub ? cbSub : pMemToMap->cb, pszTag);
1231 if (pMemDarwin)
1232 {
1233 pMemDarwin->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1234 pMemDarwin->pMemMap = pMemMap;
1235// pMemDarwin->pMemDesc = pMemDesc;
1236 *ppMem = &pMemDarwin->Core;
1237
1238 IPRT_DARWIN_RESTORE_EFL_AC();
1239 return VINF_SUCCESS;
1240 }
1241
1242// pMemDesc->complete();
1243// rc = VERR_NO_MEMORY;
1244// }
1245// else
1246// rc = RTErrConvertFromDarwinIO(IORet);
1247// pMemDesc->release();
1248// }
1249// else
1250// rc = VERR_MEMOBJ_INIT_FAILED;
1251 }
1252 else if (pv)
1253 rc = VERR_ADDRESS_TOO_BIG;
1254 else
1255 rc = VERR_MAP_FAILED;
1256 pMemMap->release();
1257 }
1258 else
1259 rc = VERR_MAP_FAILED;
1260 }
1261
1262 IPRT_DARWIN_RESTORE_EFL_AC();
1263 return rc;
1264}
1265
1266
1267DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
1268 unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub, const char *pszTag)
1269{
1270 RT_NOREF(fProt);
1271
1272 /*
1273 * Check for unsupported things.
1274 */
1275 AssertReturn(R3PtrFixed == (RTR3PTR)-1, VERR_NOT_SUPPORTED);
1276 if (uAlignment > PAGE_SIZE)
1277 return VERR_NOT_SUPPORTED;
1278 Assert(!offSub || cbSub);
1279
1280 IPRT_DARWIN_SAVE_EFL_AC();
1281
1282 /*
1283 * Must have a memory descriptor.
1284 */
1285 int rc = VERR_INVALID_PARAMETER;
1286 PRTR0MEMOBJDARWIN pMemToMapDarwin = (PRTR0MEMOBJDARWIN)pMemToMap;
1287 if (pMemToMapDarwin->pMemDesc)
1288 {
1289#if MAC_OS_X_VERSION_MIN_REQUIRED >= 101000 /* The kIOMapPrefault option was added in 10.10.0. */
1290 IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask((task_t)R0Process,
1291 0,
1292 kIOMapAnywhere | kIOMapDefaultCache | kIOMapPrefault,
1293 offSub,
1294 cbSub);
1295#elif MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
1296 static uint32_t volatile s_fOptions = UINT32_MAX;
1297 uint32_t fOptions = s_fOptions;
1298 if (RT_UNLIKELY(fOptions == UINT32_MAX))
1299 s_fOptions = fOptions = version_major >= 14 ? 0x10000000 /*kIOMapPrefault*/ : 0; /* Since 10.10.0. */
1300 IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask((task_t)R0Process,
1301 0,
1302 kIOMapAnywhere | kIOMapDefaultCache | fOptions,
1303 offSub,
1304 cbSub);
1305#else
1306 IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->map((task_t)R0Process,
1307 0,
1308 kIOMapAnywhere | kIOMapDefaultCache,
1309 offSub,
1310 cbSub);
1311#endif
1312 if (pMemMap)
1313 {
1314 IOVirtualAddress VirtAddr = pMemMap->getVirtualAddress();
1315 void *pv = (void *)(uintptr_t)VirtAddr;
1316 if ((uintptr_t)pv == VirtAddr && pv != NULL)
1317 {
1318 /*
1319 * Create the IPRT memory object.
1320 */
1321 PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_MAPPING,
1322 pv, cbSub ? cbSub : pMemToMap->cb, pszTag);
1323 if (pMemDarwin)
1324 {
1325 pMemDarwin->Core.u.Mapping.R0Process = R0Process;
1326 pMemDarwin->pMemMap = pMemMap;
1327 *ppMem = &pMemDarwin->Core;
1328
1329 IPRT_DARWIN_RESTORE_EFL_AC();
1330 return VINF_SUCCESS;
1331 }
1332
1333 rc = VERR_NO_MEMORY;
1334 }
1335 else if (pv)
1336 rc = VERR_ADDRESS_TOO_BIG;
1337 else
1338 rc = VERR_MAP_FAILED;
1339 pMemMap->release();
1340 }
1341 else
1342 rc = VERR_MAP_FAILED;
1343 }
1344
1345 IPRT_DARWIN_RESTORE_EFL_AC();
1346 return rc;
1347}
1348
1349
1350/**
1351 * Worker for rtR0MemObjNativeProtect that's typically called in a different
1352 * context.
1353 */
1354static int rtR0MemObjNativeProtectWorker(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1355{
1356 IPRT_DARWIN_SAVE_EFL_AC();
1357
1358 /* Get the map for the object. */
1359 vm_map_t pVmMap = rtR0MemObjDarwinGetMap(pMem);
1360 if (!pVmMap)
1361 {
1362 IPRT_DARWIN_RESTORE_EFL_AC();
1363 return VERR_NOT_SUPPORTED;
1364 }
1365
1366 /*
1367 * Convert the protection.
1368 */
1369 vm_prot_t fMachProt;
1370 switch (fProt)
1371 {
1372 case RTMEM_PROT_NONE:
1373 fMachProt = VM_PROT_NONE;
1374 break;
1375 case RTMEM_PROT_READ:
1376 fMachProt = VM_PROT_READ;
1377 break;
1378 case RTMEM_PROT_READ | RTMEM_PROT_WRITE:
1379 fMachProt = VM_PROT_READ | VM_PROT_WRITE;
1380 break;
1381 case RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC:
1382 fMachProt = VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
1383 break;
1384 case RTMEM_PROT_WRITE:
1385 fMachProt = VM_PROT_WRITE | VM_PROT_READ; /* never write-only */
1386 break;
1387 case RTMEM_PROT_WRITE | RTMEM_PROT_EXEC:
1388 fMachProt = VM_PROT_WRITE | VM_PROT_EXECUTE | VM_PROT_READ; /* never write-only or execute-only */
1389 break;
1390 case RTMEM_PROT_EXEC:
1391 fMachProt = VM_PROT_EXECUTE | VM_PROT_READ; /* never execute-only */
1392 break;
1393 default:
1394 AssertFailedReturn(VERR_INVALID_PARAMETER);
1395 }
1396
1397 /*
1398 * Do the job.
1399 */
1400 vm_offset_t Start = (uintptr_t)pMem->pv + offSub;
1401 kern_return_t krc = vm_protect(pVmMap,
1402 Start,
1403 cbSub,
1404 false,
1405 fMachProt);
1406 if (krc != KERN_SUCCESS)
1407 {
1408 static int s_cComplaints = 0;
1409 if (s_cComplaints < 10)
1410 {
1411 s_cComplaints++;
1412 printf("rtR0MemObjNativeProtect: vm_protect(%p,%p,%p,false,%#x) -> %d\n",
1413 (void *)pVmMap, (void *)Start, (void *)cbSub, fMachProt, krc);
1414
1415 kern_return_t krc2;
1416 vm_offset_t pvReal = Start;
1417 vm_size_t cbReal = 0;
1418 mach_msg_type_number_t cInfo = VM_REGION_BASIC_INFO_COUNT;
1419 struct vm_region_basic_info Info;
1420 RT_ZERO(Info);
1421 krc2 = vm_region(pVmMap, &pvReal, &cbReal, VM_REGION_BASIC_INFO, (vm_region_info_t)&Info, &cInfo, NULL);
1422 printf("rtR0MemObjNativeProtect: basic info - krc2=%d pv=%p cb=%p prot=%#x max=%#x inh=%#x shr=%d rvd=%d off=%#x behavior=%#x wired=%#x\n",
1423 krc2, (void *)pvReal, (void *)cbReal, Info.protection, Info.max_protection, Info.inheritance,
1424 Info.shared, Info.reserved, Info.offset, Info.behavior, Info.user_wired_count);
1425 }
1426 IPRT_DARWIN_RESTORE_EFL_AC();
1427 return RTErrConvertFromDarwinKern(krc);
1428 }
1429
1430 /*
1431 * Touch the pages if they should be writable afterwards and accessible
1432 * from code which should never fault. vm_protect() may leave pages
1433 * temporarily write protected, possibly due to pmap no-upgrade rules?
1434 *
1435 * This is the same trick (or HACK ALERT if you like) as applied in
1436 * rtR0MemObjNativeMapKernel.
1437 */
1438 if ( pMem->enmType != RTR0MEMOBJTYPE_MAPPING
1439 || pMem->u.Mapping.R0Process == NIL_RTR0PROCESS)
1440 {
1441 if (fProt & RTMEM_PROT_WRITE)
1442 rtR0MemObjDarwinTouchPages((void *)Start, cbSub);
1443 /*
1444 * Sniff (read) read-only pages too, just to be sure.
1445 */
1446 else if (fProt & (RTMEM_PROT_READ | RTMEM_PROT_EXEC))
1447 rtR0MemObjDarwinSniffPages((void const *)Start, cbSub);
1448 }
1449
1450 IPRT_DARWIN_RESTORE_EFL_AC();
1451 return VINF_SUCCESS;
1452}
1453
1454
1455/**
1456 * rtR0MemObjNativeProtect kernel_task wrapper function.
1457 */
1458static void rtR0MemObjNativeProtectWorkerOnKernelThread(void *pvUser0, void *pvUser1)
1459{
1460 AssertPtr(pvUser0); Assert(pvUser1 == NULL); NOREF(pvUser1);
1461 RTR0MEMOBJDARWINPROTECTARGS *pArgs = (RTR0MEMOBJDARWINPROTECTARGS *)pvUser0;
1462 int rc = rtR0MemObjNativeProtectWorker(pArgs->pMem, pArgs->offSub, pArgs->cbSub, pArgs->fProt);
1463 rtR0MemObjDarwinSignalThreadWaitinOnTask(&pArgs->Core, rc);
1464}
1465
1466
1467DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1468{
1469 /*
1470 * The code won't work right because process codesigning properties leaks
1471 * into kernel_map memory management. So, if the user process we're running
1472 * in has CS restrictions active, we cannot play around with the EXEC
1473 * protection because some vm_fault.c think we're modifying the process map
1474 * or something.
1475 */
1476 int rc;
1477 if (rtR0MemObjDarwinGetMap(pMem) == kernel_map)
1478 {
1479 RTR0MEMOBJDARWINPROTECTARGS Args;
1480 Args.pMem = pMem;
1481 Args.offSub = offSub;
1482 Args.cbSub = cbSub;
1483 Args.fProt = fProt;
1484 rc = rtR0MemObjDarwinDoInKernelTaskThread(rtR0MemObjNativeProtectWorkerOnKernelThread, &Args.Core);
1485 }
1486 else
1487 rc = rtR0MemObjNativeProtectWorker(pMem, offSub, cbSub, fProt);
1488 return rc;
1489}
1490
1491
1492DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1493{
1494 RTHCPHYS PhysAddr;
1495 PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)pMem;
1496 IPRT_DARWIN_SAVE_EFL_AC();
1497
1498#ifdef USE_VM_MAP_WIRE
1499 /*
1500 * Locked memory doesn't have a memory descriptor and
1501 * needs to be handled differently.
1502 */
1503 if (pMemDarwin->Core.enmType == RTR0MEMOBJTYPE_LOCK)
1504 {
1505 ppnum_t PgNo;
1506 if (pMemDarwin->Core.u.Lock.R0Process == NIL_RTR0PROCESS)
1507 PgNo = pmap_find_phys(kernel_pmap, (uintptr_t)pMemDarwin->Core.pv + iPage * PAGE_SIZE);
1508 else
1509 {
1510 /*
1511 * From what I can tell, Apple seems to have locked up the all the
1512 * available interfaces that could help us obtain the pmap_t of a task
1513 * or vm_map_t.
1514
1515 * So, we'll have to figure out where in the vm_map_t structure it is
1516 * and read it our selves. ASSUMING that kernel_pmap is pointed to by
1517 * kernel_map->pmap, we scan kernel_map to locate the structure offset.
1518 * Not nice, but it will hopefully do the job in a reliable manner...
1519 *
1520 * (get_task_pmap, get_map_pmap or vm_map_pmap is what we really need btw.)
1521 */
1522 static int s_offPmap = -1;
1523 if (RT_UNLIKELY(s_offPmap == -1))
1524 {
1525 pmap_t const *p = (pmap_t *)kernel_map;
1526 pmap_t const * const pEnd = p + 64;
1527 for (; p < pEnd; p++)
1528 if (*p == kernel_pmap)
1529 {
1530 s_offPmap = (uintptr_t)p - (uintptr_t)kernel_map;
1531 break;
1532 }
1533 AssertReturn(s_offPmap >= 0, NIL_RTHCPHYS);
1534 }
1535 pmap_t Pmap = *(pmap_t *)((uintptr_t)get_task_map((task_t)pMemDarwin->Core.u.Lock.R0Process) + s_offPmap);
1536 PgNo = pmap_find_phys(Pmap, (uintptr_t)pMemDarwin->Core.pv + iPage * PAGE_SIZE);
1537 }
1538
1539 IPRT_DARWIN_RESTORE_EFL_AC();
1540 AssertReturn(PgNo, NIL_RTHCPHYS);
1541 PhysAddr = (RTHCPHYS)PgNo << PAGE_SHIFT;
1542 Assert((PhysAddr >> PAGE_SHIFT) == PgNo);
1543 }
1544 else
1545#endif /* USE_VM_MAP_WIRE */
1546 {
1547 /*
1548 * Get the memory descriptor.
1549 */
1550 IOMemoryDescriptor *pMemDesc = pMemDarwin->pMemDesc;
1551 if (!pMemDesc)
1552 pMemDesc = pMemDarwin->pMemMap->getMemoryDescriptor();
1553 AssertReturn(pMemDesc, NIL_RTHCPHYS);
1554
1555 /*
1556 * If we've got a memory descriptor, use getPhysicalSegment64().
1557 */
1558#ifdef __LP64__
1559 addr64_t Addr = pMemDesc->getPhysicalSegment(iPage * PAGE_SIZE, NULL, kIOMemoryMapperNone);
1560#else
1561 addr64_t Addr = pMemDesc->getPhysicalSegment64(iPage * PAGE_SIZE, NULL);
1562#endif
1563 IPRT_DARWIN_RESTORE_EFL_AC();
1564 AssertMsgReturn(Addr, ("iPage=%u\n", iPage), NIL_RTHCPHYS);
1565 PhysAddr = Addr;
1566 AssertMsgReturn(PhysAddr == Addr, ("PhysAddr=%RHp Addr=%RX64\n", PhysAddr, (uint64_t)Addr), NIL_RTHCPHYS);
1567 }
1568
1569 return PhysAddr;
1570}
1571
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use