VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c

Last change on this file was 104096, checked in by vboxsync, 2 months ago

Additions: Linux/arm64: Fix VirtToPage implementation and make HGCM work, bugref:10457.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Rev Revision
File size: 74.1 KB
Line 
1/* $Id: memobj-r0drv-linux.c 104096 2024-03-27 17:38:10Z vboxsync $ */
2/** @file
3 * IPRT - Ring-0 Memory Objects, Linux.
4 */
5
6/*
7 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * The contents of this file may alternatively be used under the terms
26 * of the Common Development and Distribution License Version 1.0
27 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28 * in the VirtualBox distribution, in which case the provisions of the
29 * CDDL are applicable instead of those of the GPL.
30 *
31 * You may elect to license modified versions of this file under the
32 * terms and conditions of either the GPL or the CDDL or both.
33 *
34 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35 */
36
37
38/*********************************************************************************************************************************
39* Header Files *
40*********************************************************************************************************************************/
41#include "the-linux-kernel.h"
42
43#include <iprt/memobj.h>
44#include <iprt/assert.h>
45#include <iprt/err.h>
46#include <iprt/log.h>
47#include <iprt/mem.h>
48#include <iprt/process.h>
49#include <iprt/string.h>
50#include "internal/memobj.h"
51#include "internal/iprt.h"
52
53
54/*********************************************************************************************************************************
55* Defined Constants And Macros *
56*********************************************************************************************************************************/
57/* early 2.6 kernels */
58#ifndef PAGE_SHARED_EXEC
59# define PAGE_SHARED_EXEC PAGE_SHARED
60#endif
61#ifndef PAGE_READONLY_EXEC
62# define PAGE_READONLY_EXEC PAGE_READONLY
63#endif
64
65/** @def IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
66 * Whether we use alloc_vm_area (3.2+) for executable memory.
67 * This is a must for 5.8+, but we enable it all the way back to 3.2.x for
68 * better W^R compliance (fExecutable flag). */
69#if RTLNX_VER_RANGE(3,2,0, 5,10,0) || defined(DOXYGEN_RUNNING)
70# define IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
71#endif
72/** @def IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
73 * alloc_vm_area was removed with 5.10 so we have to resort to a different way
74 * to allocate executable memory.
75 * It would be possible to remove IPRT_USE_ALLOC_VM_AREA_FOR_EXEC and use
76 * this path execlusively for 3.2+ but no time to test it really works on every
77 * supported kernel, so better play safe for now.
78 */
79#if RTLNX_VER_MIN(5,10,0) || defined(DOXYGEN_RUNNING)
80# define IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
81#endif
82
83/*
84 * 2.6.29+ kernels don't work with remap_pfn_range() anymore because
85 * track_pfn_vma_new() is apparently not defined for non-RAM pages.
86 * It should be safe to use vm_insert_page() older kernels as well.
87 */
88#if RTLNX_VER_MIN(2,6,23)
89# define VBOX_USE_INSERT_PAGE
90#endif
91#if defined(CONFIG_X86_PAE) \
92 && ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
93 || RTLNX_VER_RANGE(2,6,0, 2,6,11) )
94# define VBOX_USE_PAE_HACK
95#endif
96
97/* gfp_t was introduced in 2.6.14, define it for earlier. */
98#if RTLNX_VER_MAX(2,6,14)
99# define gfp_t unsigned
100#endif
101
102/*
103 * Wrappers around mmap_lock/mmap_sem difference.
104 */
105#if RTLNX_VER_MIN(5,8,0)
106# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_lock)
107# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_lock)
108# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_lock)
109# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_lock)
110#else
111# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_sem)
112# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_sem)
113# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_sem)
114# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_sem)
115#endif
116
117
118/*********************************************************************************************************************************
119* Structures and Typedefs *
120*********************************************************************************************************************************/
121/**
122 * The Linux version of the memory object structure.
123 */
124typedef struct RTR0MEMOBJLNX
125{
126 /** The core structure. */
127 RTR0MEMOBJINTERNAL Core;
128 /** Set if the allocation is contiguous.
129 * This means it has to be given back as one chunk. */
130 bool fContiguous;
131 /** Set if executable allocation. */
132 bool fExecutable;
133 /** Set if we've vmap'ed the memory into ring-0. */
134 bool fMappedToRing0;
135 /** This is non-zero if large page allocation. */
136 uint8_t cLargePageOrder;
137#ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
138 /** Return from alloc_vm_area() that we now need to use for executable
139 * memory. */
140 struct vm_struct *pArea;
141 /** PTE array that goes along with pArea (must be freed). */
142 pte_t **papPtesForArea;
143#endif
144 /** The pages in the apPages array. */
145 size_t cPages;
146 /** Array of struct page pointers. (variable size) */
147 RT_FLEXIBLE_ARRAY_EXTENSION
148 struct page *apPages[RT_FLEXIBLE_ARRAY];
149} RTR0MEMOBJLNX;
150/** Pointer to the linux memory object. */
151typedef RTR0MEMOBJLNX *PRTR0MEMOBJLNX;
152
153
154/*********************************************************************************************************************************
155* Global Variables *
156*********************************************************************************************************************************/
157/*
158 * Linux allows only a coarse selection of zones for
159 * allocations matching a particular maximum physical address.
160 *
161 * Sorted from high to low physical address!
162 */
163static const struct
164{
165 RTHCPHYS PhysHighest;
166 gfp_t fGfp;
167} g_aZones[] =
168{
169 { NIL_RTHCPHYS, GFP_KERNEL },
170#if (defined(RT_ARCH_AMD64) || defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
171 { _4G - 1, GFP_DMA32 }, /* ZONE_DMA32: 0-4GB */
172#elif defined(RT_ARCH_ARM32) || defined(RT_ARCH_ARM64)
173 { _4G - 1, GFP_DMA }, /* ZONE_DMA: 0-4GB */
174#endif
175#if defined(RT_ARCH_AMD64)
176 { _16M - 1, GFP_DMA }, /* ZONE_DMA: 0-16MB */
177#elif defined(RT_ARCH_X86)
178 { 896 * _1M - 1, GFP_USER }, /* ZONE_NORMAL (32-bit hosts): 0-896MB */
179#endif
180};
181
182
183static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx);
184
185
186/**
187 * Helper that converts from a RTR0PROCESS handle to a linux task.
188 *
189 * @returns The corresponding Linux task.
190 * @param R0Process IPRT ring-0 process handle.
191 */
192static struct task_struct *rtR0ProcessToLinuxTask(RTR0PROCESS R0Process)
193{
194 /** @todo fix rtR0ProcessToLinuxTask!! */
195 /** @todo many (all?) callers currently assume that we return 'current'! */
196 return R0Process == RTR0ProcHandleSelf() ? current : NULL;
197}
198
199
200/**
201 * Compute order. Some functions allocate 2^order pages.
202 *
203 * @returns order.
204 * @param cPages Number of pages.
205 */
206static int rtR0MemObjLinuxOrder(size_t cPages)
207{
208 int iOrder;
209 size_t cTmp;
210
211 for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
212 ;
213 if (cPages & ~((size_t)1 << iOrder))
214 ++iOrder;
215
216 return iOrder;
217}
218
219
220/**
221 * Converts from RTMEM_PROT_* to Linux PAGE_*.
222 *
223 * @returns Linux page protection constant.
224 * @param fProt The IPRT protection mask.
225 * @param fKernel Whether it applies to kernel or user space.
226 */
227static pgprot_t rtR0MemObjLinuxConvertProt(unsigned fProt, bool fKernel)
228{
229 switch (fProt)
230 {
231 default:
232 AssertMsgFailed(("%#x %d\n", fProt, fKernel)); RT_FALL_THRU();
233 case RTMEM_PROT_NONE:
234 return PAGE_NONE;
235
236 case RTMEM_PROT_READ:
237 return fKernel ? PAGE_KERNEL_RO : PAGE_READONLY;
238
239 case RTMEM_PROT_WRITE:
240 case RTMEM_PROT_WRITE | RTMEM_PROT_READ:
241 return fKernel ? PAGE_KERNEL : PAGE_SHARED;
242
243 case RTMEM_PROT_EXEC:
244 case RTMEM_PROT_EXEC | RTMEM_PROT_READ:
245#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
246 if (fKernel)
247 {
248# if RTLNX_VER_MIN(6,6,0)
249 /* In kernel 6.6 mk_pte() macro was fortified with additional
250 * check which does not allow to use our custom mask anymore
251 * (see kernel commit ae1f05a617dcbc0a732fbeba0893786cd009536c).
252 * For this particular mapping case, an existing mask PAGE_KERNEL_ROX
253 * can be used instead. PAGE_KERNEL_ROX was introduced in
254 * kernel 5.8, however, lets apply it for kernels 6.6 and newer
255 * to be on a safe side.
256 */
257 return PAGE_KERNEL_ROX;
258# else
259 pgprot_t fPg = MY_PAGE_KERNEL_EXEC;
260 pgprot_val(fPg) &= ~_PAGE_RW;
261 return fPg;
262# endif
263 }
264 return PAGE_READONLY_EXEC;
265#else
266 return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_READONLY_EXEC;
267#endif
268
269 case RTMEM_PROT_WRITE | RTMEM_PROT_EXEC:
270 case RTMEM_PROT_WRITE | RTMEM_PROT_EXEC | RTMEM_PROT_READ:
271 return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_SHARED_EXEC;
272 }
273}
274
275
276/**
277 * Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
278 * an empty user space mapping.
279 *
280 * We acquire the mmap_sem/mmap_lock of the task!
281 *
282 * @returns Pointer to the mapping.
283 * (void *)-1 on failure.
284 * @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
285 * @param cb The size of the mapping.
286 * @param uAlignment The alignment of the mapping.
287 * @param pTask The Linux task to create this mapping in.
288 * @param fProt The RTMEM_PROT_* mask.
289 */
290static void *rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, struct task_struct *pTask, unsigned fProt)
291{
292 unsigned fLnxProt;
293 unsigned long ulAddr;
294
295 Assert(pTask == current); /* do_mmap */
296 RT_NOREF_PV(pTask);
297
298 /*
299 * Convert from IPRT protection to mman.h PROT_ and call do_mmap.
300 */
301 fProt &= (RTMEM_PROT_NONE | RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC);
302 if (fProt == RTMEM_PROT_NONE)
303 fLnxProt = PROT_NONE;
304 else
305 {
306 fLnxProt = 0;
307 if (fProt & RTMEM_PROT_READ)
308 fLnxProt |= PROT_READ;
309 if (fProt & RTMEM_PROT_WRITE)
310 fLnxProt |= PROT_WRITE;
311 if (fProt & RTMEM_PROT_EXEC)
312 fLnxProt |= PROT_EXEC;
313 }
314
315 if (R3PtrFixed != (RTR3PTR)-1)
316 {
317#if RTLNX_VER_MIN(3,5,0)
318 ulAddr = vm_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, 0);
319#else
320 LNX_MM_DOWN_WRITE(pTask->mm);
321 ulAddr = do_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, 0);
322 LNX_MM_UP_WRITE(pTask->mm);
323#endif
324 }
325 else
326 {
327#if RTLNX_VER_MIN(3,5,0)
328 ulAddr = vm_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS, 0);
329#else
330 LNX_MM_DOWN_WRITE(pTask->mm);
331 ulAddr = do_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS, 0);
332 LNX_MM_UP_WRITE(pTask->mm);
333#endif
334 if ( !(ulAddr & ~PAGE_MASK)
335 && (ulAddr & (uAlignment - 1)))
336 {
337 /** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
338 * up alignment gaps. This is of course complicated by fragmentation (which we might have cause
339 * ourselves) and further by there begin two mmap strategies (top / bottom). */
340 /* For now, just ignore uAlignment requirements... */
341 }
342 }
343
344
345 if (ulAddr & ~PAGE_MASK) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
346 return (void *)-1;
347 return (void *)ulAddr;
348}
349
350
351/**
352 * Worker that destroys a user space mapping.
353 * Undoes what rtR0MemObjLinuxDoMmap did.
354 *
355 * We acquire the mmap_sem/mmap_lock of the task!
356 *
357 * @param pv The ring-3 mapping.
358 * @param cb The size of the mapping.
359 * @param pTask The Linux task to destroy this mapping in.
360 */
361static void rtR0MemObjLinuxDoMunmap(void *pv, size_t cb, struct task_struct *pTask)
362{
363#if RTLNX_VER_MIN(3,5,0)
364 Assert(pTask == current); RT_NOREF_PV(pTask);
365 vm_munmap((unsigned long)pv, cb);
366#elif defined(USE_RHEL4_MUNMAP)
367 LNX_MM_DOWN_WRITE(pTask->mm);
368 do_munmap(pTask->mm, (unsigned long)pv, cb, 0); /* should it be 1 or 0? */
369 LNX_MM_UP_WRITE(pTask->mm);
370#else
371 LNX_MM_DOWN_WRITE(pTask->mm);
372 do_munmap(pTask->mm, (unsigned long)pv, cb);
373 LNX_MM_UP_WRITE(pTask->mm);
374#endif
375}
376
377
378/**
379 * Internal worker that allocates physical pages and creates the memory object for them.
380 *
381 * @returns IPRT status code.
382 * @param ppMemLnx Where to store the memory object pointer.
383 * @param enmType The object type.
384 * @param cb The number of bytes to allocate.
385 * @param uAlignment The alignment of the physical memory.
386 * Only valid if fContiguous == true, ignored otherwise.
387 * @param fFlagsLnx The page allocation flags (GPFs).
388 * @param fContiguous Whether the allocation must be contiguous.
389 * @param fExecutable Whether the memory must be executable.
390 * @param rcNoMem What to return when we're out of pages.
391 * @param pszTag Allocation tag used for statistics and such.
392 */
393static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX *ppMemLnx, RTR0MEMOBJTYPE enmType, size_t cb,
394 size_t uAlignment, gfp_t fFlagsLnx, bool fContiguous, bool fExecutable, int rcNoMem,
395 const char *pszTag)
396{
397 size_t iPage;
398 size_t const cPages = cb >> PAGE_SHIFT;
399 struct page *paPages;
400
401 /*
402 * Allocate a memory object structure that's large enough to contain
403 * the page pointer array.
404 */
405 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), enmType,
406 NULL, cb, pszTag);
407 if (!pMemLnx)
408 return VERR_NO_MEMORY;
409 pMemLnx->Core.fFlags |= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC;
410 pMemLnx->cPages = cPages;
411
412 if (cPages > 255)
413 {
414# ifdef __GFP_REPEAT
415 /* Try hard to allocate the memory, but the allocation attempt might fail. */
416 fFlagsLnx |= __GFP_REPEAT;
417# endif
418# ifdef __GFP_NOMEMALLOC
419 /* Introduced with Linux 2.6.12: Don't use emergency reserves */
420 fFlagsLnx |= __GFP_NOMEMALLOC;
421# endif
422 }
423
424 /*
425 * Allocate the pages.
426 * For small allocations we'll try contiguous first and then fall back on page by page.
427 */
428#if RTLNX_VER_MIN(2,4,22)
429 if ( fContiguous
430 || cb <= PAGE_SIZE * 2)
431 {
432# ifdef VBOX_USE_INSERT_PAGE
433 paPages = alloc_pages(fFlagsLnx | __GFP_COMP | __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
434# else
435 paPages = alloc_pages(fFlagsLnx | __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
436# endif
437 if (paPages)
438 {
439 fContiguous = true;
440 for (iPage = 0; iPage < cPages; iPage++)
441 pMemLnx->apPages[iPage] = &paPages[iPage];
442 }
443 else if (fContiguous)
444 {
445 rtR0MemObjDelete(&pMemLnx->Core);
446 return rcNoMem;
447 }
448 }
449
450 if (!fContiguous)
451 {
452 /** @todo Try use alloc_pages_bulk_array when available, it should be faster
453 * than a alloc_page loop. Put it in #ifdefs similar to
454 * IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC. */
455 for (iPage = 0; iPage < cPages; iPage++)
456 {
457 pMemLnx->apPages[iPage] = alloc_page(fFlagsLnx | __GFP_NOWARN);
458 if (RT_UNLIKELY(!pMemLnx->apPages[iPage]))
459 {
460 while (iPage-- > 0)
461 __free_page(pMemLnx->apPages[iPage]);
462 rtR0MemObjDelete(&pMemLnx->Core);
463 return rcNoMem;
464 }
465 }
466 }
467
468#else /* < 2.4.22 */
469 /** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
470 paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
471 if (!paPages)
472 {
473 rtR0MemObjDelete(&pMemLnx->Core);
474 return rcNoMem;
475 }
476 for (iPage = 0; iPage < cPages; iPage++)
477 {
478 pMemLnx->apPages[iPage] = &paPages[iPage];
479 if (fExecutable)
480 MY_SET_PAGES_EXEC(pMemLnx->apPages[iPage], 1);
481 if (PageHighMem(pMemLnx->apPages[iPage]))
482 BUG();
483 }
484
485 fContiguous = true;
486#endif /* < 2.4.22 */
487 pMemLnx->fContiguous = fContiguous;
488 pMemLnx->fExecutable = fExecutable;
489
490#if RTLNX_VER_MAX(4,5,0)
491 /*
492 * Reserve the pages.
493 *
494 * Linux >= 4.5 with CONFIG_DEBUG_VM panics when setting PG_reserved on compound
495 * pages. According to Michal Hocko this shouldn't be necessary anyway because
496 * as pages which are not on the LRU list are never evictable.
497 */
498 for (iPage = 0; iPage < cPages; iPage++)
499 SetPageReserved(pMemLnx->apPages[iPage]);
500#endif
501
502 /*
503 * Note that the physical address of memory allocated with alloc_pages(flags, order)
504 * is always 2^(PAGE_SHIFT+order)-aligned.
505 */
506 if ( fContiguous
507 && uAlignment > PAGE_SIZE)
508 {
509 /*
510 * Check for alignment constraints. The physical address of memory allocated with
511 * alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
512 */
513 if (RT_UNLIKELY(page_to_phys(pMemLnx->apPages[0]) & (uAlignment - 1)))
514 {
515 /*
516 * This should never happen!
517 */
518 printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
519 (unsigned long)cb, (unsigned long)uAlignment, rtR0MemObjLinuxOrder(cPages), (unsigned long)page_to_phys(pMemLnx->apPages[0]));
520 rtR0MemObjLinuxFreePages(pMemLnx);
521 return rcNoMem;
522 }
523 }
524
525 *ppMemLnx = pMemLnx;
526 return VINF_SUCCESS;
527}
528
529
530/**
531 * Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
532 *
533 * This method does NOT free the object.
534 *
535 * @param pMemLnx The object which physical pages should be freed.
536 */
537static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx)
538{
539 size_t iPage = pMemLnx->cPages;
540 if (iPage > 0)
541 {
542 /*
543 * Restore the page flags.
544 */
545 while (iPage-- > 0)
546 {
547#if RTLNX_VER_MAX(4,5,0)
548 /* See SetPageReserved() in rtR0MemObjLinuxAllocPages() */
549 ClearPageReserved(pMemLnx->apPages[iPage]);
550#endif
551#if RTLNX_VER_MAX(2,4,22)
552 if (pMemLnx->fExecutable)
553 MY_SET_PAGES_NOEXEC(pMemLnx->apPages[iPage], 1);
554#endif
555 }
556
557 /*
558 * Free the pages.
559 */
560#if RTLNX_VER_MIN(2,4,22)
561 if (!pMemLnx->fContiguous)
562 {
563 iPage = pMemLnx->cPages;
564 while (iPage-- > 0)
565 __free_page(pMemLnx->apPages[iPage]);
566 }
567 else
568#endif
569 __free_pages(pMemLnx->apPages[0], rtR0MemObjLinuxOrder(pMemLnx->cPages));
570
571 pMemLnx->cPages = 0;
572 }
573}
574
575
576#ifdef IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
577/**
578 * User data passed to the apply_to_page_range() callback.
579 */
580typedef struct LNXAPPLYPGRANGE
581{
582 /** Pointer to the memory object. */
583 PRTR0MEMOBJLNX pMemLnx;
584 /** The page protection flags to apply. */
585 pgprot_t fPg;
586} LNXAPPLYPGRANGE;
587/** Pointer to the user data. */
588typedef LNXAPPLYPGRANGE *PLNXAPPLYPGRANGE;
589/** Pointer to the const user data. */
590typedef const LNXAPPLYPGRANGE *PCLNXAPPLYPGRANGE;
591
592/**
593 * Callback called in apply_to_page_range().
594 *
595 * @returns Linux status code.
596 * @param pPte Pointer to the page table entry for the given address.
597 * @param uAddr The address to apply the new protection to.
598 * @param pvUser The opaque user data.
599 */
600static int rtR0MemObjLinuxApplyPageRange(pte_t *pPte, unsigned long uAddr, void *pvUser)
601{
602 PCLNXAPPLYPGRANGE pArgs = (PCLNXAPPLYPGRANGE)pvUser;
603 PRTR0MEMOBJLNX pMemLnx = pArgs->pMemLnx;
604 size_t idxPg = (uAddr - (unsigned long)pMemLnx->Core.pv) >> PAGE_SHIFT;
605
606 set_pte(pPte, mk_pte(pMemLnx->apPages[idxPg], pArgs->fPg));
607 return 0;
608}
609#endif
610
611
612/**
613 * Maps the allocation into ring-0.
614 *
615 * This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
616 *
617 * Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
618 * space, so we'll use that mapping if possible. If execute access is required, we'll
619 * play safe and do our own mapping.
620 *
621 * @returns IPRT status code.
622 * @param pMemLnx The linux memory object to map.
623 * @param fExecutable Whether execute access is required.
624 */
625static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx, bool fExecutable)
626{
627 int rc = VINF_SUCCESS;
628
629 /*
630 * Choose mapping strategy.
631 */
632 bool fMustMap = fExecutable
633 || !pMemLnx->fContiguous;
634 if (!fMustMap)
635 {
636 size_t iPage = pMemLnx->cPages;
637 while (iPage-- > 0)
638 if (PageHighMem(pMemLnx->apPages[iPage]))
639 {
640 fMustMap = true;
641 break;
642 }
643 }
644
645 Assert(!pMemLnx->Core.pv);
646 Assert(!pMemLnx->fMappedToRing0);
647
648 if (fMustMap)
649 {
650 /*
651 * Use vmap - 2.4.22 and later.
652 */
653#if RTLNX_VER_MIN(2,4,22) && (defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86))
654 pgprot_t fPg;
655 pgprot_val(fPg) = _PAGE_PRESENT | _PAGE_RW;
656# ifdef _PAGE_NX
657 if (!fExecutable)
658 pgprot_val(fPg) |= _PAGE_NX;
659# endif
660
661# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
662 if (fExecutable)
663 {
664# if RTLNX_VER_MIN(3,2,51)
665 pte_t **papPtes = (pte_t **)kmalloc_array(pMemLnx->cPages, sizeof(papPtes[0]), GFP_KERNEL);
666# else
667 pte_t **papPtes = (pte_t **)kmalloc(pMemLnx->cPages * sizeof(papPtes[0]), GFP_KERNEL);
668# endif
669 if (papPtes)
670 {
671 pMemLnx->pArea = alloc_vm_area(pMemLnx->Core.cb, papPtes); /* Note! pArea->nr_pages is not set. */
672 if (pMemLnx->pArea)
673 {
674 size_t i;
675 Assert(pMemLnx->pArea->size >= pMemLnx->Core.cb); /* Note! includes guard page. */
676 Assert(pMemLnx->pArea->addr);
677# ifdef _PAGE_NX
678 pgprot_val(fPg) |= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
679# endif
680 pMemLnx->papPtesForArea = papPtes;
681 for (i = 0; i < pMemLnx->cPages; i++)
682 *papPtes[i] = mk_pte(pMemLnx->apPages[i], fPg);
683 pMemLnx->Core.pv = pMemLnx->pArea->addr;
684 pMemLnx->fMappedToRing0 = true;
685 }
686 else
687 {
688 kfree(papPtes);
689 rc = VERR_MAP_FAILED;
690 }
691 }
692 else
693 rc = VERR_MAP_FAILED;
694 }
695 else
696# endif
697 {
698# if defined(IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC)
699 if (fExecutable)
700 pgprot_val(fPg) |= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
701# endif
702
703# ifdef VM_MAP
704 pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_MAP, fPg);
705# else
706 pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_ALLOC, fPg);
707# endif
708 if (pMemLnx->Core.pv)
709 pMemLnx->fMappedToRing0 = true;
710 else
711 rc = VERR_MAP_FAILED;
712 }
713#else /* < 2.4.22 */
714 rc = VERR_NOT_SUPPORTED;
715#endif
716 }
717 else
718 {
719 /*
720 * Use the kernel RAM mapping.
721 */
722 pMemLnx->Core.pv = phys_to_virt(page_to_phys(pMemLnx->apPages[0]));
723 Assert(pMemLnx->Core.pv);
724 }
725
726 return rc;
727}
728
729
730/**
731 * Undoes what rtR0MemObjLinuxVMap() did.
732 *
733 * @param pMemLnx The linux memory object.
734 */
735static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx)
736{
737#if RTLNX_VER_MIN(2,4,22)
738# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
739 if (pMemLnx->pArea)
740 {
741# if 0
742 pte_t **papPtes = pMemLnx->papPtesForArea;
743 size_t i;
744 for (i = 0; i < pMemLnx->cPages; i++)
745 *papPtes[i] = 0;
746# endif
747 free_vm_area(pMemLnx->pArea);
748 kfree(pMemLnx->papPtesForArea);
749 pMemLnx->pArea = NULL;
750 pMemLnx->papPtesForArea = NULL;
751 }
752 else
753# endif
754 if (pMemLnx->fMappedToRing0)
755 {
756 Assert(pMemLnx->Core.pv);
757 vunmap(pMemLnx->Core.pv);
758 pMemLnx->fMappedToRing0 = false;
759 }
760#else /* < 2.4.22 */
761 Assert(!pMemLnx->fMappedToRing0);
762#endif
763 pMemLnx->Core.pv = NULL;
764}
765
766
767DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
768{
769 IPRT_LINUX_SAVE_EFL_AC();
770 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
771
772 /*
773 * Release any memory that we've allocated or locked.
774 */
775 switch (pMemLnx->Core.enmType)
776 {
777 case RTR0MEMOBJTYPE_PAGE:
778 case RTR0MEMOBJTYPE_LOW:
779 case RTR0MEMOBJTYPE_CONT:
780 case RTR0MEMOBJTYPE_PHYS:
781 case RTR0MEMOBJTYPE_PHYS_NC:
782 rtR0MemObjLinuxVUnmap(pMemLnx);
783 rtR0MemObjLinuxFreePages(pMemLnx);
784 break;
785
786 case RTR0MEMOBJTYPE_LARGE_PAGE:
787 {
788 uint32_t const cLargePages = pMemLnx->Core.cb >> (pMemLnx->cLargePageOrder + PAGE_SHIFT);
789 uint32_t iLargePage;
790 for (iLargePage = 0; iLargePage < cLargePages; iLargePage++)
791 __free_pages(pMemLnx->apPages[iLargePage << pMemLnx->cLargePageOrder], pMemLnx->cLargePageOrder);
792 pMemLnx->cPages = 0;
793
794#ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
795 Assert(!pMemLnx->pArea);
796 Assert(!pMemLnx->papPtesForArea);
797#endif
798 break;
799 }
800
801 case RTR0MEMOBJTYPE_LOCK:
802 if (pMemLnx->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
803 {
804 struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
805 size_t iPage;
806 Assert(pTask);
807 if (pTask && pTask->mm)
808 LNX_MM_DOWN_READ(pTask->mm);
809
810 iPage = pMemLnx->cPages;
811 while (iPage-- > 0)
812 {
813 if (!PageReserved(pMemLnx->apPages[iPage]))
814 SetPageDirty(pMemLnx->apPages[iPage]);
815#if RTLNX_VER_MIN(4,6,0)
816 put_page(pMemLnx->apPages[iPage]);
817#else
818 page_cache_release(pMemLnx->apPages[iPage]);
819#endif
820 }
821
822 if (pTask && pTask->mm)
823 LNX_MM_UP_READ(pTask->mm);
824 }
825 /* else: kernel memory - nothing to do here. */
826 break;
827
828 case RTR0MEMOBJTYPE_RES_VIRT:
829 Assert(pMemLnx->Core.pv);
830 if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
831 {
832 struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
833 Assert(pTask);
834 if (pTask && pTask->mm)
835 rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
836 }
837 else
838 {
839 vunmap(pMemLnx->Core.pv);
840
841 Assert(pMemLnx->cPages == 1 && pMemLnx->apPages[0] != NULL);
842 __free_page(pMemLnx->apPages[0]);
843 pMemLnx->apPages[0] = NULL;
844 pMemLnx->cPages = 0;
845 }
846 pMemLnx->Core.pv = NULL;
847 break;
848
849 case RTR0MEMOBJTYPE_MAPPING:
850 Assert(pMemLnx->cPages == 0); Assert(pMemLnx->Core.pv);
851 if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
852 {
853 struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
854 Assert(pTask);
855 if (pTask && pTask->mm)
856 rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
857 }
858 else
859 vunmap(pMemLnx->Core.pv);
860 pMemLnx->Core.pv = NULL;
861 break;
862
863 default:
864 AssertMsgFailed(("enmType=%d\n", pMemLnx->Core.enmType));
865 return VERR_INTERNAL_ERROR;
866 }
867 IPRT_LINUX_RESTORE_EFL_ONLY_AC();
868 return VINF_SUCCESS;
869}
870
871
872DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
873{
874 IPRT_LINUX_SAVE_EFL_AC();
875 PRTR0MEMOBJLNX pMemLnx;
876 int rc;
877
878#if RTLNX_VER_MIN(2,4,22)
879 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_HIGHUSER,
880 false /* non-contiguous */, fExecutable, VERR_NO_MEMORY, pszTag);
881#else
882 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_USER,
883 false /* non-contiguous */, fExecutable, VERR_NO_MEMORY, pszTag);
884#endif
885 if (RT_SUCCESS(rc))
886 {
887 rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
888 if (RT_SUCCESS(rc))
889 {
890 *ppMem = &pMemLnx->Core;
891 IPRT_LINUX_RESTORE_EFL_AC();
892 return rc;
893 }
894
895 rtR0MemObjLinuxFreePages(pMemLnx);
896 rtR0MemObjDelete(&pMemLnx->Core);
897 }
898
899 IPRT_LINUX_RESTORE_EFL_AC();
900 return rc;
901}
902
903
904DECLHIDDEN(int) rtR0MemObjNativeAllocLarge(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, size_t cbLargePage, uint32_t fFlags,
905 const char *pszTag)
906{
907#ifdef GFP_TRANSHUGE
908 /*
909 * Allocate a memory object structure that's large enough to contain
910 * the page pointer array.
911 */
912# ifdef __GFP_MOVABLE
913 unsigned const fGfp = (GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE;
914# else
915 unsigned const fGfp = (GFP_TRANSHUGE | __GFP_ZERO);
916# endif
917 size_t const cPagesPerLarge = cbLargePage >> PAGE_SHIFT;
918 unsigned const cLargePageOrder = rtR0MemObjLinuxOrder(cPagesPerLarge);
919 size_t const cLargePages = cb >> (cLargePageOrder + PAGE_SHIFT);
920 size_t const cPages = cb >> PAGE_SHIFT;
921 PRTR0MEMOBJLNX pMemLnx;
922
923 Assert(RT_BIT_64(cLargePageOrder + PAGE_SHIFT) == cbLargePage);
924 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]),
925 RTR0MEMOBJTYPE_LARGE_PAGE, NULL, cb, pszTag);
926 if (pMemLnx)
927 {
928 size_t iLargePage;
929
930 pMemLnx->Core.fFlags |= RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC;
931 pMemLnx->cLargePageOrder = cLargePageOrder;
932 pMemLnx->cPages = cPages;
933
934 /*
935 * Allocate the requested number of large pages.
936 */
937 for (iLargePage = 0; iLargePage < cLargePages; iLargePage++)
938 {
939 struct page *paPages = alloc_pages(fGfp, cLargePageOrder);
940 if (paPages)
941 {
942 size_t const iPageBase = iLargePage << cLargePageOrder;
943 size_t iPage = cPagesPerLarge;
944 while (iPage-- > 0)
945 pMemLnx->apPages[iPageBase + iPage] = &paPages[iPage];
946 }
947 else
948 {
949 /*Log(("rtR0MemObjNativeAllocLarge: cb=%#zx cPages=%#zx cLargePages=%#zx cLargePageOrder=%u cPagesPerLarge=%#zx iLargePage=%#zx -> failed!\n",
950 cb, cPages, cLargePages, cLargePageOrder, cPagesPerLarge, iLargePage, paPages));*/
951 while (iLargePage-- > 0)
952 __free_pages(pMemLnx->apPages[iLargePage << (cLargePageOrder - PAGE_SHIFT)], cLargePageOrder);
953 rtR0MemObjDelete(&pMemLnx->Core);
954 return VERR_NO_MEMORY;
955 }
956 }
957 *ppMem = &pMemLnx->Core;
958 return VINF_SUCCESS;
959 }
960 return VERR_NO_MEMORY;
961
962#else
963 /*
964 * We don't call rtR0MemObjFallbackAllocLarge here as it can be a really
965 * bad idea to trigger the swap daemon and whatnot. So, just fail.
966 */
967 RT_NOREF(ppMem, cb, cbLargePage, fFlags, pszTag);
968 return VERR_NOT_SUPPORTED;
969#endif
970}
971
972
973DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
974{
975 IPRT_LINUX_SAVE_EFL_AC();
976 PRTR0MEMOBJLNX pMemLnx;
977 int rc;
978
979 /* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
980#if (defined(RT_ARCH_AMD64) || defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
981 /* ZONE_DMA32: 0-4GB */
982 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA32,
983 false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY, pszTag);
984 if (RT_FAILURE(rc))
985#endif
986#ifdef RT_ARCH_AMD64
987 /* ZONE_DMA: 0-16MB */
988 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA,
989 false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY, pszTag);
990#else
991# ifdef CONFIG_X86_PAE
992# endif
993 /* ZONE_NORMAL: 0-896MB */
994 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_USER,
995 false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY, pszTag);
996#endif
997 if (RT_SUCCESS(rc))
998 {
999 rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
1000 if (RT_SUCCESS(rc))
1001 {
1002 *ppMem = &pMemLnx->Core;
1003 IPRT_LINUX_RESTORE_EFL_AC();
1004 return rc;
1005 }
1006
1007 rtR0MemObjLinuxFreePages(pMemLnx);
1008 rtR0MemObjDelete(&pMemLnx->Core);
1009 }
1010
1011 IPRT_LINUX_RESTORE_EFL_AC();
1012 return rc;
1013}
1014
1015
1016DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest,
1017 bool fExecutable, const char *pszTag)
1018{
1019 IPRT_LINUX_SAVE_EFL_AC();
1020 PRTR0MEMOBJLNX pMemLnx;
1021 int rc;
1022 uint32_t idxZone;
1023
1024 /*
1025 * The last zone must be able to satisfy the PhysHighest requirement or there
1026 * will be no zone at all.
1027 */
1028 if (g_aZones[RT_ELEMENTS(g_aZones) - 1].PhysHighest > PhysHighest)
1029 {
1030 IPRT_LINUX_RESTORE_EFL_AC();
1031 AssertMsgFailedReturn(("No zone can satisfy PhysHighest=%RHp!\n", PhysHighest),
1032 VERR_NO_CONT_MEMORY);
1033 }
1034
1035 /* Find the first zone matching our PhysHighest requirement. */
1036 idxZone = 0;
1037 for (;;)
1038 {
1039 if (g_aZones[idxZone].PhysHighest <= PhysHighest)
1040 break; /* We found a zone satisfying the requirement. */
1041 idxZone++;
1042 }
1043
1044 /* Now try to allocate pages from all the left zones until one succeeds. */
1045 for (;;)
1046 {
1047 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, g_aZones[idxZone].fGfp,
1048 true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY, pszTag);
1049 idxZone++;
1050 if (RT_SUCCESS(rc) || idxZone == RT_ELEMENTS(g_aZones))
1051 break;
1052 }
1053 if (RT_SUCCESS(rc))
1054 {
1055 rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
1056 if (RT_SUCCESS(rc))
1057 {
1058#if defined(RT_STRICT)
1059 size_t iPage = pMemLnx->cPages;
1060 while (iPage-- > 0)
1061 Assert(page_to_phys(pMemLnx->apPages[iPage]) < PhysHighest);
1062#endif
1063 pMemLnx->Core.u.Cont.Phys = page_to_phys(pMemLnx->apPages[0]);
1064 *ppMem = &pMemLnx->Core;
1065 IPRT_LINUX_RESTORE_EFL_AC();
1066 return rc;
1067 }
1068
1069 rtR0MemObjLinuxFreePages(pMemLnx);
1070 rtR0MemObjDelete(&pMemLnx->Core);
1071 }
1072
1073 IPRT_LINUX_RESTORE_EFL_AC();
1074 return rc;
1075}
1076
1077
1078/**
1079 * Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
1080 *
1081 * @returns IPRT status code.
1082 * @param ppMemLnx Where to
1083 * @param enmType The object type.
1084 * @param cb The size of the allocation.
1085 * @param uAlignment The alignment of the physical memory.
1086 * Only valid for fContiguous == true, ignored otherwise.
1087 * @param PhysHighest See rtR0MemObjNativeAllocPhys.
1088 * @param pszTag Allocation tag used for statistics and such.
1089 * @param fGfp The Linux GFP flags to use for the allocation.
1090 */
1091static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
1092 size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, const char *pszTag, gfp_t fGfp)
1093{
1094 PRTR0MEMOBJLNX pMemLnx;
1095 int rc = rtR0MemObjLinuxAllocPages(&pMemLnx, enmType, cb, uAlignment, fGfp,
1096 enmType == RTR0MEMOBJTYPE_PHYS /* contiguous / non-contiguous */,
1097 false /*fExecutable*/, VERR_NO_PHYS_MEMORY, pszTag);
1098 if (RT_FAILURE(rc))
1099 return rc;
1100
1101 /*
1102 * Check the addresses if necessary. (Can be optimized a bit for PHYS.)
1103 */
1104 if (PhysHighest != NIL_RTHCPHYS)
1105 {
1106 size_t iPage = pMemLnx->cPages;
1107 while (iPage-- > 0)
1108 if (page_to_phys(pMemLnx->apPages[iPage]) > PhysHighest)
1109 {
1110 rtR0MemObjLinuxFreePages(pMemLnx);
1111 rtR0MemObjDelete(&pMemLnx->Core);
1112 return VERR_NO_MEMORY;
1113 }
1114 }
1115
1116 /*
1117 * Complete the object.
1118 */
1119 if (enmType == RTR0MEMOBJTYPE_PHYS)
1120 {
1121 pMemLnx->Core.u.Phys.PhysBase = page_to_phys(pMemLnx->apPages[0]);
1122 pMemLnx->Core.u.Phys.fAllocated = true;
1123 }
1124 *ppMem = &pMemLnx->Core;
1125 return rc;
1126}
1127
1128
1129/**
1130 * Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
1131 *
1132 * @returns IPRT status code.
1133 * @param ppMem Where to store the memory object pointer on success.
1134 * @param enmType The object type.
1135 * @param cb The size of the allocation.
1136 * @param uAlignment The alignment of the physical memory.
1137 * Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
1138 * @param PhysHighest See rtR0MemObjNativeAllocPhys.
1139 * @param pszTag Allocation tag used for statistics and such.
1140 */
1141static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
1142 size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, const char *pszTag)
1143{
1144 int rc;
1145 IPRT_LINUX_SAVE_EFL_AC();
1146
1147 /*
1148 * There are two clear cases and that's the <=16MB and anything-goes ones.
1149 * When the physical address limit is somewhere in-between those two we'll
1150 * just have to try, starting with HIGHUSER and working our way thru the
1151 * different types, hoping we'll get lucky.
1152 *
1153 * We should probably move this physical address restriction logic up to
1154 * the page alloc function as it would be more efficient there. But since
1155 * we don't expect this to be a performance issue just yet it can wait.
1156 */
1157 if (PhysHighest == NIL_RTHCPHYS)
1158 /* ZONE_HIGHMEM: the whole physical memory */
1159 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_HIGHUSER);
1160 else if (PhysHighest <= _1M * 16)
1161 /* ZONE_DMA: 0-16MB */
1162 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_DMA);
1163 else
1164 {
1165 rc = VERR_NO_MEMORY;
1166 if (RT_FAILURE(rc))
1167 /* ZONE_HIGHMEM: the whole physical memory */
1168 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_HIGHUSER);
1169 if (RT_FAILURE(rc))
1170 /* ZONE_NORMAL: 0-896MB */
1171 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_USER);
1172#ifdef GFP_DMA32
1173 if (RT_FAILURE(rc))
1174 /* ZONE_DMA32: 0-4GB */
1175 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_DMA32);
1176#endif
1177 if (RT_FAILURE(rc))
1178 /* ZONE_DMA: 0-16MB */
1179 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_DMA);
1180 }
1181 IPRT_LINUX_RESTORE_EFL_AC();
1182 return rc;
1183}
1184
1185
1186/**
1187 * Translates a kernel virtual address to a linux page structure by walking the
1188 * page tables.
1189 *
1190 * @note We do assume that the page tables will not change as we are walking
1191 * them. This assumption is rather forced by the fact that I could not
1192 * immediately see any way of preventing this from happening. So, we
1193 * take some extra care when accessing them.
1194 *
1195 * Because of this, we don't want to use this function on memory where
1196 * attribute changes to nearby pages is likely to cause large pages to
1197 * be used or split up. So, don't use this for the linear mapping of
1198 * physical memory.
1199 *
1200 * @returns Pointer to the page structur or NULL if it could not be found.
1201 * @param pv The kernel virtual address.
1202 */
1203RTDECL(struct page *) rtR0MemObjLinuxVirtToPage(void *pv)
1204{
1205#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
1206 unsigned long ulAddr = (unsigned long)pv;
1207 unsigned long pfn;
1208 struct page *pPage;
1209 pte_t *pEntry;
1210 union
1211 {
1212 pgd_t Global;
1213# if RTLNX_VER_MIN(4,12,0)
1214 p4d_t Four;
1215# endif
1216# if RTLNX_VER_MIN(2,6,11)
1217 pud_t Upper;
1218# endif
1219 pmd_t Middle;
1220 pte_t Entry;
1221 } u;
1222
1223 /* Should this happen in a situation this code will be called in? And if
1224 * so, can it change under our feet? See also
1225 * "Documentation/vm/active_mm.txt" in the kernel sources. */
1226 if (RT_UNLIKELY(!current->active_mm))
1227 return NULL;
1228 u.Global = *pgd_offset(current->active_mm, ulAddr);
1229 if (RT_UNLIKELY(pgd_none(u.Global)))
1230 return NULL;
1231# if RTLNX_VER_MIN(2,6,11)
1232# if RTLNX_VER_MIN(4,12,0)
1233 u.Four = *p4d_offset(&u.Global, ulAddr);
1234 if (RT_UNLIKELY(p4d_none(u.Four)))
1235 return NULL;
1236# if RTLNX_VER_MIN(5,6,0)
1237 if (p4d_leaf(u.Four))
1238# else
1239 if (p4d_large(u.Four))
1240# endif
1241 {
1242 pPage = p4d_page(u.Four);
1243 AssertReturn(pPage, NULL);
1244 pfn = page_to_pfn(pPage); /* doing the safe way... */
1245 AssertCompile(P4D_SHIFT - PAGE_SHIFT < 31);
1246 pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (P4D_SHIFT - PAGE_SHIFT)) - 1);
1247 return pfn_to_page(pfn);
1248 }
1249 u.Upper = *pud_offset(&u.Four, ulAddr);
1250# else /* < 4.12 */
1251 u.Upper = *pud_offset(&u.Global, ulAddr);
1252# endif /* < 4.12 */
1253 if (RT_UNLIKELY(pud_none(u.Upper)))
1254 return NULL;
1255# if RTLNX_VER_MIN(2,6,25)
1256# if RTLNX_VER_MIN(5,6,0)
1257 if (pud_leaf(u.Upper))
1258# else
1259 if (pud_large(u.Upper))
1260# endif
1261 {
1262 pPage = pud_page(u.Upper);
1263 AssertReturn(pPage, NULL);
1264 pfn = page_to_pfn(pPage); /* doing the safe way... */
1265 pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PUD_SHIFT - PAGE_SHIFT)) - 1);
1266 return pfn_to_page(pfn);
1267 }
1268# endif
1269 u.Middle = *pmd_offset(&u.Upper, ulAddr);
1270# else /* < 2.6.11 */
1271 u.Middle = *pmd_offset(&u.Global, ulAddr);
1272# endif /* < 2.6.11 */
1273 if (RT_UNLIKELY(pmd_none(u.Middle)))
1274 return NULL;
1275# if RTLNX_VER_MIN(2,6,0)
1276# if RTLNX_VER_MIN(5,6,0)
1277 if (pmd_leaf(u.Middle))
1278# else
1279 if (pmd_large(u.Middle))
1280# endif
1281 {
1282 pPage = pmd_page(u.Middle);
1283 AssertReturn(pPage, NULL);
1284 pfn = page_to_pfn(pPage); /* doing the safe way... */
1285 pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PMD_SHIFT - PAGE_SHIFT)) - 1);
1286 return pfn_to_page(pfn);
1287 }
1288# endif
1289
1290# if RTLNX_VER_MIN(6,5,0) || RTLNX_RHEL_RANGE(9,4, 9,99)
1291 pEntry = __pte_map(&u.Middle, ulAddr);
1292# elif RTLNX_VER_MIN(2,5,5) || defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
1293 pEntry = pte_offset_map(&u.Middle, ulAddr);
1294# else
1295 pEntry = pte_offset(&u.Middle, ulAddr);
1296# endif
1297 if (RT_UNLIKELY(!pEntry))
1298 return NULL;
1299 u.Entry = *pEntry;
1300# if RTLNX_VER_MIN(2,5,5) || defined(pte_offset_map)
1301 pte_unmap(pEntry);
1302# endif
1303
1304 if (RT_UNLIKELY(!pte_present(u.Entry)))
1305 return NULL;
1306 return pte_page(u.Entry);
1307#else /* !defined(RT_ARCH_AMD64) && !defined(RT_ARCH_X86) */
1308
1309 if (is_vmalloc_addr(pv))
1310 return vmalloc_to_page(pv);
1311
1312 return virt_to_page(pv);
1313#endif
1314}
1315RT_EXPORT_SYMBOL(rtR0MemObjLinuxVirtToPage);
1316
1317
1318DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment,
1319 const char *pszTag)
1320{
1321 return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS, cb, uAlignment, PhysHighest, pszTag);
1322}
1323
1324
1325DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, const char *pszTag)
1326{
1327 return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS_NC, cb, PAGE_SIZE, PhysHighest, pszTag);
1328}
1329
1330
1331DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy,
1332 const char *pszTag)
1333{
1334 IPRT_LINUX_SAVE_EFL_AC();
1335
1336 /*
1337 * All we need to do here is to validate that we can use
1338 * ioremap on the specified address (32/64-bit dma_addr_t).
1339 */
1340 PRTR0MEMOBJLNX pMemLnx;
1341 dma_addr_t PhysAddr = Phys;
1342 AssertMsgReturn(PhysAddr == Phys, ("%#llx\n", (unsigned long long)Phys), VERR_ADDRESS_TOO_BIG);
1343
1344 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_PHYS, NULL, cb, pszTag);
1345 if (!pMemLnx)
1346 {
1347 IPRT_LINUX_RESTORE_EFL_AC();
1348 return VERR_NO_MEMORY;
1349 }
1350
1351 pMemLnx->Core.u.Phys.PhysBase = PhysAddr;
1352 pMemLnx->Core.u.Phys.fAllocated = false;
1353 pMemLnx->Core.u.Phys.uCachePolicy = uCachePolicy;
1354 Assert(!pMemLnx->cPages);
1355 *ppMem = &pMemLnx->Core;
1356 IPRT_LINUX_RESTORE_EFL_AC();
1357 return VINF_SUCCESS;
1358}
1359
1360/* openSUSE Leap 42.3 detection :-/ */
1361#if RTLNX_VER_RANGE(4,4,0, 4,6,0) && defined(FAULT_FLAG_REMOTE)
1362# define GET_USER_PAGES_API KERNEL_VERSION(4, 10, 0) /* no typo! */
1363#else
1364# define GET_USER_PAGES_API LINUX_VERSION_CODE
1365#endif
1366
1367DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess,
1368 RTR0PROCESS R0Process, const char *pszTag)
1369{
1370 IPRT_LINUX_SAVE_EFL_AC();
1371 const int cPages = cb >> PAGE_SHIFT;
1372 struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1373# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
1374 struct vm_area_struct **papVMAs;
1375# endif
1376 PRTR0MEMOBJLNX pMemLnx;
1377 int rc = VERR_NO_MEMORY;
1378 int const fWrite = fAccess & RTMEM_PROT_WRITE ? 1 : 0;
1379
1380 /*
1381 * Check for valid task and size overflows.
1382 */
1383 if (!pTask)
1384 return VERR_NOT_SUPPORTED;
1385 if (((size_t)cPages << PAGE_SHIFT) != cb)
1386 return VERR_OUT_OF_RANGE;
1387
1388 /*
1389 * Allocate the memory object and a temporary buffer for the VMAs.
1390 */
1391 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK,
1392 (void *)R3Ptr, cb, pszTag);
1393 if (!pMemLnx)
1394 {
1395 IPRT_LINUX_RESTORE_EFL_AC();
1396 return VERR_NO_MEMORY;
1397 }
1398
1399# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
1400 papVMAs = (struct vm_area_struct **)RTMemAlloc(sizeof(*papVMAs) * cPages);
1401 if (papVMAs)
1402 {
1403# endif
1404 LNX_MM_DOWN_READ(pTask->mm);
1405
1406 /*
1407 * Get user pages.
1408 */
1409/** @todo r=bird: Should we not force read access too? */
1410#if GET_USER_PAGES_API >= KERNEL_VERSION(4, 6, 0)
1411 if (R0Process == RTR0ProcHandleSelf())
1412 rc = get_user_pages(R3Ptr, /* Where from. */
1413 cPages, /* How many pages. */
1414# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1415 fWrite ? FOLL_WRITE | /* Write to memory. */
1416 FOLL_FORCE /* force write access. */
1417 : 0, /* Write to memory. */
1418# else
1419 fWrite, /* Write to memory. */
1420 fWrite, /* force write access. */
1421# endif
1422 &pMemLnx->apPages[0] /* Page array. */
1423# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
1424 , papVMAs /* vmas */
1425# endif
1426 );
1427 /*
1428 * Actually this should not happen at the moment as call this function
1429 * only for our own process.
1430 */
1431 else
1432 rc = get_user_pages_remote(
1433# if GET_USER_PAGES_API < KERNEL_VERSION(5, 9, 0)
1434 pTask, /* Task for fault accounting. */
1435# endif
1436 pTask->mm, /* Whose pages. */
1437 R3Ptr, /* Where from. */
1438 cPages, /* How many pages. */
1439# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1440 fWrite ? FOLL_WRITE | /* Write to memory. */
1441 FOLL_FORCE /* force write access. */
1442 : 0, /* Write to memory. */
1443# else
1444 fWrite, /* Write to memory. */
1445 fWrite, /* force write access. */
1446# endif
1447 &pMemLnx->apPages[0] /* Page array. */
1448# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
1449 , papVMAs /* vmas */
1450# endif
1451# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 10, 0)
1452 , NULL /* locked */
1453# endif
1454 );
1455#else /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1456 rc = get_user_pages(pTask, /* Task for fault accounting. */
1457 pTask->mm, /* Whose pages. */
1458 R3Ptr, /* Where from. */
1459 cPages, /* How many pages. */
1460/* The get_user_pages API change was back-ported to 4.4.168. */
1461# if RTLNX_VER_RANGE(4,4,168, 4,5,0)
1462 fWrite ? FOLL_WRITE | /* Write to memory. */
1463 FOLL_FORCE /* force write access. */
1464 : 0, /* Write to memory. */
1465# else
1466 fWrite, /* Write to memory. */
1467 fWrite, /* force write access. */
1468# endif
1469 &pMemLnx->apPages[0] /* Page array. */
1470# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
1471 , papVMAs /* vmas */
1472# endif
1473 );
1474#endif /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1475 if (rc == cPages)
1476 {
1477 /*
1478 * Flush dcache (required?), protect against fork and _really_ pin the page
1479 * table entries. get_user_pages() will protect against swapping out the
1480 * pages but it will NOT protect against removing page table entries. This
1481 * can be achieved with
1482 * - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
1483 * an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
1484 * Usual Linux distributions support only a limited size of locked pages
1485 * (e.g. 32KB).
1486 * - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
1487 * or by
1488 * - setting the VM_LOCKED flag. This is the same as doing mlock() without
1489 * a range check.
1490 */
1491 /** @todo The Linux fork() protection will require more work if this API
1492 * is to be used for anything but locking VM pages. */
1493 while (rc-- > 0)
1494 {
1495 flush_dcache_page(pMemLnx->apPages[rc]);
1496# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
1497# if RTLNX_VER_MIN(6,3,0)
1498 vm_flags_set(papVMAs[rc], VM_DONTCOPY | VM_LOCKED);
1499# else
1500 papVMAs[rc]->vm_flags |= VM_DONTCOPY | VM_LOCKED;
1501# endif
1502# endif
1503 }
1504
1505 LNX_MM_UP_READ(pTask->mm);
1506
1507# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
1508 RTMemFree(papVMAs);
1509# endif
1510
1511 pMemLnx->Core.u.Lock.R0Process = R0Process;
1512 pMemLnx->cPages = cPages;
1513 Assert(!pMemLnx->fMappedToRing0);
1514 *ppMem = &pMemLnx->Core;
1515
1516 IPRT_LINUX_RESTORE_EFL_AC();
1517 return VINF_SUCCESS;
1518 }
1519
1520 /*
1521 * Failed - we need to unlock any pages that we succeeded to lock.
1522 */
1523 while (rc-- > 0)
1524 {
1525 if (!PageReserved(pMemLnx->apPages[rc]))
1526 SetPageDirty(pMemLnx->apPages[rc]);
1527#if RTLNX_VER_MIN(4,6,0)
1528 put_page(pMemLnx->apPages[rc]);
1529#else
1530 page_cache_release(pMemLnx->apPages[rc]);
1531#endif
1532 }
1533
1534 LNX_MM_UP_READ(pTask->mm);
1535
1536 rc = VERR_LOCK_FAILED;
1537
1538# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
1539 RTMemFree(papVMAs);
1540 }
1541# endif
1542
1543 rtR0MemObjDelete(&pMemLnx->Core);
1544 IPRT_LINUX_RESTORE_EFL_AC();
1545 return rc;
1546}
1547
1548
1549DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess, const char *pszTag)
1550{
1551 IPRT_LINUX_SAVE_EFL_AC();
1552 void *pvLast = (uint8_t *)pv + cb - 1;
1553 size_t const cPages = cb >> PAGE_SHIFT;
1554 PRTR0MEMOBJLNX pMemLnx;
1555 bool fLinearMapping;
1556 int rc;
1557 uint8_t *pbPage;
1558 size_t iPage;
1559 NOREF(fAccess);
1560
1561 if ( !RTR0MemKernelIsValidAddr(pv)
1562 || !RTR0MemKernelIsValidAddr(pv + cb))
1563 return VERR_INVALID_PARAMETER;
1564
1565 /*
1566 * The lower part of the kernel memory has a linear mapping between
1567 * physical and virtual addresses. So we take a short cut here. This is
1568 * assumed to be the cleanest way to handle those addresses (and the code
1569 * is well tested, though the test for determining it is not very nice).
1570 * If we ever decide it isn't we can still remove it.
1571 */
1572#if 0
1573 fLinearMapping = (unsigned long)pvLast < VMALLOC_START;
1574#else
1575 fLinearMapping = (unsigned long)pv >= (unsigned long)__va(0)
1576 && (unsigned long)pvLast < (unsigned long)high_memory;
1577#endif
1578
1579 /*
1580 * Allocate the memory object.
1581 */
1582 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK,
1583 pv, cb, pszTag);
1584 if (!pMemLnx)
1585 {
1586 IPRT_LINUX_RESTORE_EFL_AC();
1587 return VERR_NO_MEMORY;
1588 }
1589
1590 /*
1591 * Gather the pages.
1592 * We ASSUME all kernel pages are non-swappable and non-movable.
1593 */
1594 rc = VINF_SUCCESS;
1595 pbPage = (uint8_t *)pvLast;
1596 iPage = cPages;
1597 if (!fLinearMapping)
1598 {
1599 while (iPage-- > 0)
1600 {
1601 struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1602 if (RT_UNLIKELY(!pPage))
1603 {
1604 rc = VERR_LOCK_FAILED;
1605 break;
1606 }
1607 pMemLnx->apPages[iPage] = pPage;
1608 pbPage -= PAGE_SIZE;
1609 }
1610 }
1611 else
1612 {
1613 while (iPage-- > 0)
1614 {
1615 pMemLnx->apPages[iPage] = virt_to_page(pbPage);
1616 pbPage -= PAGE_SIZE;
1617 }
1618 }
1619 if (RT_SUCCESS(rc))
1620 {
1621 /*
1622 * Complete the memory object and return.
1623 */
1624 pMemLnx->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
1625 pMemLnx->cPages = cPages;
1626 Assert(!pMemLnx->fMappedToRing0);
1627 *ppMem = &pMemLnx->Core;
1628
1629 IPRT_LINUX_RESTORE_EFL_AC();
1630 return VINF_SUCCESS;
1631 }
1632
1633 rtR0MemObjDelete(&pMemLnx->Core);
1634 IPRT_LINUX_RESTORE_EFL_AC();
1635 return rc;
1636}
1637
1638
1639DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment,
1640 const char *pszTag)
1641{
1642#if RTLNX_VER_MIN(2,4,22)
1643 IPRT_LINUX_SAVE_EFL_AC();
1644 const size_t cPages = cb >> PAGE_SHIFT;
1645 struct page *pDummyPage;
1646 struct page **papPages;
1647
1648 /* check for unsupported stuff. */
1649 AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1650 if (uAlignment > PAGE_SIZE)
1651 return VERR_NOT_SUPPORTED;
1652
1653 /*
1654 * Allocate a dummy page and create a page pointer array for vmap such that
1655 * the dummy page is mapped all over the reserved area.
1656 */
1657 pDummyPage = alloc_page(GFP_HIGHUSER | __GFP_NOWARN);
1658 if (pDummyPage)
1659 {
1660 papPages = RTMemAlloc(sizeof(*papPages) * cPages);
1661 if (papPages)
1662 {
1663 void *pv;
1664 size_t iPage = cPages;
1665 while (iPage-- > 0)
1666 papPages[iPage] = pDummyPage;
1667# ifdef VM_MAP
1668 pv = vmap(papPages, cPages, VM_MAP, PAGE_KERNEL_RO);
1669# else
1670 pv = vmap(papPages, cPages, VM_ALLOC, PAGE_KERNEL_RO);
1671# endif
1672 RTMemFree(papPages);
1673 if (pv)
1674 {
1675 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb, pszTag);
1676 if (pMemLnx)
1677 {
1678 pMemLnx->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
1679 pMemLnx->cPages = 1;
1680 pMemLnx->apPages[0] = pDummyPage;
1681 *ppMem = &pMemLnx->Core;
1682 IPRT_LINUX_RESTORE_EFL_AC();
1683 return VINF_SUCCESS;
1684 }
1685 vunmap(pv);
1686 }
1687 }
1688 __free_page(pDummyPage);
1689 }
1690 IPRT_LINUX_RESTORE_EFL_AC();
1691 return VERR_NO_MEMORY;
1692
1693#else /* < 2.4.22 */
1694 /*
1695 * Could probably use ioremap here, but the caller is in a better position than us
1696 * to select some safe physical memory.
1697 */
1698 return VERR_NOT_SUPPORTED;
1699#endif
1700}
1701
1702
1703DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment,
1704 RTR0PROCESS R0Process, const char *pszTag)
1705{
1706 IPRT_LINUX_SAVE_EFL_AC();
1707 PRTR0MEMOBJLNX pMemLnx;
1708 void *pv;
1709 struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1710 if (!pTask)
1711 return VERR_NOT_SUPPORTED;
1712
1713 /*
1714 * Check that the specified alignment is supported.
1715 */
1716 if (uAlignment > PAGE_SIZE)
1717 return VERR_NOT_SUPPORTED;
1718
1719 /*
1720 * Let rtR0MemObjLinuxDoMmap do the difficult bits.
1721 */
1722 pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cb, uAlignment, pTask, RTMEM_PROT_NONE);
1723 if (pv == (void *)-1)
1724 {
1725 IPRT_LINUX_RESTORE_EFL_AC();
1726 return VERR_NO_MEMORY;
1727 }
1728
1729 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb, pszTag);
1730 if (!pMemLnx)
1731 {
1732 rtR0MemObjLinuxDoMunmap(pv, cb, pTask);
1733 IPRT_LINUX_RESTORE_EFL_AC();
1734 return VERR_NO_MEMORY;
1735 }
1736
1737 pMemLnx->Core.u.ResVirt.R0Process = R0Process;
1738 *ppMem = &pMemLnx->Core;
1739 IPRT_LINUX_RESTORE_EFL_AC();
1740 return VINF_SUCCESS;
1741}
1742
1743
1744DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment,
1745 unsigned fProt, size_t offSub, size_t cbSub, const char *pszTag)
1746{
1747 int rc = VERR_NO_MEMORY;
1748 PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1749 PRTR0MEMOBJLNX pMemLnx;
1750 IPRT_LINUX_SAVE_EFL_AC();
1751
1752 /* Fail if requested to do something we can't. */
1753 AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1754 if (uAlignment > PAGE_SIZE)
1755 return VERR_NOT_SUPPORTED;
1756
1757 /*
1758 * Create the IPRT memory object.
1759 */
1760 if (!cbSub)
1761 cbSub = pMemLnxToMap->Core.cb - offSub;
1762 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub, pszTag);
1763 if (pMemLnx)
1764 {
1765 if (pMemLnxToMap->cPages)
1766 {
1767#if RTLNX_VER_MIN(2,4,22)
1768 /*
1769 * Use vmap - 2.4.22 and later.
1770 */
1771 pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, true /* kernel */);
1772 /** @todo We don't really care too much for EXEC here... 5.8 always adds NX. */
1773 Assert(((offSub + cbSub) >> PAGE_SHIFT) <= pMemLnxToMap->cPages);
1774# ifdef VM_MAP
1775 pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_MAP, fPg);
1776# else
1777 pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_ALLOC, fPg);
1778# endif
1779 if (pMemLnx->Core.pv)
1780 {
1781 pMemLnx->fMappedToRing0 = true;
1782 rc = VINF_SUCCESS;
1783 }
1784 else
1785 rc = VERR_MAP_FAILED;
1786
1787#else /* < 2.4.22 */
1788 /*
1789 * Only option here is to share mappings if possible and forget about fProt.
1790 */
1791 if (rtR0MemObjIsRing3(pMemToMap))
1792 rc = VERR_NOT_SUPPORTED;
1793 else
1794 {
1795 rc = VINF_SUCCESS;
1796 if (!pMemLnxToMap->Core.pv)
1797 rc = rtR0MemObjLinuxVMap(pMemLnxToMap, !!(fProt & RTMEM_PROT_EXEC));
1798 if (RT_SUCCESS(rc))
1799 {
1800 Assert(pMemLnxToMap->Core.pv);
1801 pMemLnx->Core.pv = (uint8_t *)pMemLnxToMap->Core.pv + offSub;
1802 }
1803 }
1804#endif
1805 }
1806 else
1807 {
1808 /*
1809 * MMIO / physical memory.
1810 */
1811 Assert(pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS && !pMemLnxToMap->Core.u.Phys.fAllocated);
1812#if RTLNX_VER_MIN(2,6,25)
1813 /*
1814 * ioremap() defaults to no caching since the 2.6 kernels.
1815 * ioremap_nocache() has been removed finally in 5.6-rc1.
1816 */
1817 pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1818 ? ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1819 : ioremap_cache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1820#else /* KERNEL_VERSION < 2.6.25 */
1821 pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1822 ? ioremap_nocache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1823 : ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1824#endif /* KERNEL_VERSION < 2.6.25 */
1825 if (pMemLnx->Core.pv)
1826 {
1827 /** @todo fix protection. */
1828 rc = VINF_SUCCESS;
1829 }
1830 }
1831 if (RT_SUCCESS(rc))
1832 {
1833 pMemLnx->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1834 *ppMem = &pMemLnx->Core;
1835 IPRT_LINUX_RESTORE_EFL_AC();
1836 return VINF_SUCCESS;
1837 }
1838 rtR0MemObjDelete(&pMemLnx->Core);
1839 }
1840
1841 IPRT_LINUX_RESTORE_EFL_AC();
1842 return rc;
1843}
1844
1845
1846#ifdef VBOX_USE_PAE_HACK
1847/**
1848 * Replace the PFN of a PTE with the address of the actual page.
1849 *
1850 * The caller maps a reserved dummy page at the address with the desired access
1851 * and flags.
1852 *
1853 * This hack is required for older Linux kernels which don't provide
1854 * remap_pfn_range().
1855 *
1856 * @returns 0 on success, -ENOMEM on failure.
1857 * @param mm The memory context.
1858 * @param ulAddr The mapping address.
1859 * @param Phys The physical address of the page to map.
1860 */
1861static int rtR0MemObjLinuxFixPte(struct mm_struct *mm, unsigned long ulAddr, RTHCPHYS Phys)
1862{
1863 int rc = -ENOMEM;
1864 pgd_t *pgd;
1865
1866 spin_lock(&mm->page_table_lock);
1867
1868 pgd = pgd_offset(mm, ulAddr);
1869 if (!pgd_none(*pgd) && !pgd_bad(*pgd))
1870 {
1871 pmd_t *pmd = pmd_offset(pgd, ulAddr);
1872 if (!pmd_none(*pmd))
1873 {
1874 pte_t *ptep = pte_offset_map(pmd, ulAddr);
1875 if (ptep)
1876 {
1877 pte_t pte = *ptep;
1878 pte.pte_high &= 0xfff00000;
1879 pte.pte_high |= ((Phys >> 32) & 0x000fffff);
1880 pte.pte_low &= 0x00000fff;
1881 pte.pte_low |= (Phys & 0xfffff000);
1882 set_pte(ptep, pte);
1883 pte_unmap(ptep);
1884 rc = 0;
1885 }
1886 }
1887 }
1888
1889 spin_unlock(&mm->page_table_lock);
1890 return rc;
1891}
1892#endif /* VBOX_USE_PAE_HACK */
1893
1894
1895DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
1896 unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub, const char *pszTag)
1897{
1898 struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1899 PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1900 int rc = VERR_NO_MEMORY;
1901 PRTR0MEMOBJLNX pMemLnx;
1902#ifdef VBOX_USE_PAE_HACK
1903 struct page *pDummyPage;
1904 RTHCPHYS DummyPhys;
1905#endif
1906 IPRT_LINUX_SAVE_EFL_AC();
1907
1908 /*
1909 * Check for restrictions.
1910 */
1911 if (!pTask)
1912 return VERR_NOT_SUPPORTED;
1913 if (uAlignment > PAGE_SIZE)
1914 return VERR_NOT_SUPPORTED;
1915
1916#ifdef VBOX_USE_PAE_HACK
1917 /*
1918 * Allocate a dummy page for use when mapping the memory.
1919 */
1920 pDummyPage = alloc_page(GFP_USER | __GFP_NOWARN);
1921 if (!pDummyPage)
1922 {
1923 IPRT_LINUX_RESTORE_EFL_AC();
1924 return VERR_NO_MEMORY;
1925 }
1926 SetPageReserved(pDummyPage);
1927 DummyPhys = page_to_phys(pDummyPage);
1928#endif
1929
1930 /*
1931 * Create the IPRT memory object.
1932 */
1933 Assert(!offSub || cbSub);
1934 if (cbSub == 0)
1935 cbSub = pMemLnxToMap->Core.cb;
1936 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub, pszTag);
1937 if (pMemLnx)
1938 {
1939 /*
1940 * Allocate user space mapping.
1941 */
1942 void *pv;
1943 pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cbSub, uAlignment, pTask, fProt);
1944 if (pv != (void *)-1)
1945 {
1946 /*
1947 * Map page by page into the mmap area.
1948 * This is generic, paranoid and not very efficient.
1949 */
1950 pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, false /* user */);
1951 unsigned long ulAddrCur = (unsigned long)pv;
1952 const size_t cPages = (offSub + cbSub) >> PAGE_SHIFT;
1953 size_t iPage;
1954
1955 LNX_MM_DOWN_WRITE(pTask->mm);
1956
1957 rc = VINF_SUCCESS;
1958 if (pMemLnxToMap->cPages)
1959 {
1960 for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE)
1961 {
1962#if RTLNX_VER_MAX(2,6,11)
1963 RTHCPHYS Phys = page_to_phys(pMemLnxToMap->apPages[iPage]);
1964#endif
1965#if RTLNX_VER_MIN(2,6,0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1966 struct vm_area_struct *vma = find_vma(pTask->mm, ulAddrCur); /* this is probably the same for all the pages... */
1967 AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1968#endif
1969#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
1970 /* remap_page_range() limitation on x86 */
1971 AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1972#endif
1973
1974#if defined(VBOX_USE_INSERT_PAGE) && RTLNX_VER_MIN(2,6,22)
1975 rc = vm_insert_page(vma, ulAddrCur, pMemLnxToMap->apPages[iPage]);
1976 /* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
1977 * See remap_pfn_range() in mm/memory.c */
1978
1979#if RTLNX_VER_MIN(6,3,0)
1980 vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
1981#elif RTLNX_VER_MIN(3,7,0)
1982 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
1983#else
1984 vma->vm_flags |= VM_RESERVED;
1985#endif
1986#elif RTLNX_VER_MIN(2,6,11)
1987 rc = remap_pfn_range(vma, ulAddrCur, page_to_pfn(pMemLnxToMap->apPages[iPage]), PAGE_SIZE, fPg);
1988#elif defined(VBOX_USE_PAE_HACK)
1989 rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1990 if (!rc)
1991 rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1992#elif RTLNX_VER_MIN(2,6,0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1993 rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1994#else /* 2.4 */
1995 rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1996#endif
1997 if (rc)
1998 {
1999 rc = VERR_NO_MEMORY;
2000 break;
2001 }
2002 }
2003 }
2004 else
2005 {
2006 RTHCPHYS Phys;
2007 if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS)
2008 Phys = pMemLnxToMap->Core.u.Phys.PhysBase;
2009 else if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_CONT)
2010 Phys = pMemLnxToMap->Core.u.Cont.Phys;
2011 else
2012 {
2013 AssertMsgFailed(("%d\n", pMemLnxToMap->Core.enmType));
2014 Phys = NIL_RTHCPHYS;
2015 }
2016 if (Phys != NIL_RTHCPHYS)
2017 {
2018 for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE, Phys += PAGE_SIZE)
2019 {
2020#if RTLNX_VER_MIN(2,6,0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
2021 struct vm_area_struct *vma = find_vma(pTask->mm, ulAddrCur); /* this is probably the same for all the pages... */
2022 AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
2023#endif
2024#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
2025 /* remap_page_range() limitation on x86 */
2026 AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
2027#endif
2028
2029#if RTLNX_VER_MIN(2,6,11)
2030 rc = remap_pfn_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
2031#elif defined(VBOX_USE_PAE_HACK)
2032 rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
2033 if (!rc)
2034 rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
2035#elif RTLNX_VER_MIN(2,6,0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
2036 rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
2037#else /* 2.4 */
2038 rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
2039#endif
2040 if (rc)
2041 {
2042 rc = VERR_NO_MEMORY;
2043 break;
2044 }
2045 }
2046 }
2047 }
2048
2049#ifdef CONFIG_NUMA_BALANCING
2050# if RTLNX_VER_MAX(3,13,0) && RTLNX_RHEL_MAX(7,0)
2051# define VBOX_NUMA_HACK_OLD
2052# endif
2053 if (RT_SUCCESS(rc))
2054 {
2055 /** @todo Ugly hack! But right now we have no other means to
2056 * disable automatic NUMA page balancing. */
2057# ifdef RT_OS_X86
2058# ifdef VBOX_NUMA_HACK_OLD
2059 pTask->mm->numa_next_reset = jiffies + 0x7fffffffUL;
2060# endif
2061 pTask->mm->numa_next_scan = jiffies + 0x7fffffffUL;
2062# else
2063# ifdef VBOX_NUMA_HACK_OLD
2064 pTask->mm->numa_next_reset = jiffies + 0x7fffffffffffffffUL;
2065# endif
2066 pTask->mm->numa_next_scan = jiffies + 0x7fffffffffffffffUL;
2067# endif
2068 }
2069#endif /* CONFIG_NUMA_BALANCING */
2070
2071 LNX_MM_UP_WRITE(pTask->mm);
2072
2073 if (RT_SUCCESS(rc))
2074 {
2075#ifdef VBOX_USE_PAE_HACK
2076 __free_page(pDummyPage);
2077#endif
2078 pMemLnx->Core.pv = pv;
2079 pMemLnx->Core.u.Mapping.R0Process = R0Process;
2080 *ppMem = &pMemLnx->Core;
2081 IPRT_LINUX_RESTORE_EFL_AC();
2082 return VINF_SUCCESS;
2083 }
2084
2085 /*
2086 * Bail out.
2087 */
2088 rtR0MemObjLinuxDoMunmap(pv, cbSub, pTask);
2089 }
2090 rtR0MemObjDelete(&pMemLnx->Core);
2091 }
2092#ifdef VBOX_USE_PAE_HACK
2093 __free_page(pDummyPage);
2094#endif
2095
2096 IPRT_LINUX_RESTORE_EFL_AC();
2097 return rc;
2098}
2099
2100
2101DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
2102{
2103# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
2104 /*
2105 * Currently only supported when we've got addresses PTEs from the kernel.
2106 */
2107 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
2108 if (pMemLnx->pArea && pMemLnx->papPtesForArea)
2109 {
2110 pgprot_t const fPg = rtR0MemObjLinuxConvertProt(fProt, true /*fKernel*/);
2111 size_t const cPages = (offSub + cbSub) >> PAGE_SHIFT;
2112 pte_t **papPtes = pMemLnx->papPtesForArea;
2113 size_t i;
2114
2115 for (i = offSub >> PAGE_SHIFT; i < cPages; i++)
2116 {
2117 set_pte(papPtes[i], mk_pte(pMemLnx->apPages[i], fPg));
2118 }
2119 preempt_disable();
2120 __flush_tlb_all();
2121 preempt_enable();
2122 return VINF_SUCCESS;
2123 }
2124# elif defined(IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC)
2125 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
2126 if ( pMemLnx->fExecutable
2127 && pMemLnx->fMappedToRing0)
2128 {
2129 LNXAPPLYPGRANGE Args;
2130 Args.pMemLnx = pMemLnx;
2131 Args.fPg = rtR0MemObjLinuxConvertProt(fProt, true /*fKernel*/);
2132 int rcLnx = apply_to_page_range(current->active_mm, (unsigned long)pMemLnx->Core.pv + offSub, cbSub,
2133 rtR0MemObjLinuxApplyPageRange, (void *)&Args);
2134 if (rcLnx)
2135 return VERR_NOT_SUPPORTED;
2136
2137 return VINF_SUCCESS;
2138 }
2139# endif
2140
2141 NOREF(pMem);
2142 NOREF(offSub);
2143 NOREF(cbSub);
2144 NOREF(fProt);
2145 return VERR_NOT_SUPPORTED;
2146}
2147
2148
2149DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
2150{
2151 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
2152
2153 if (pMemLnx->cPages)
2154 return page_to_phys(pMemLnx->apPages[iPage]);
2155
2156 switch (pMemLnx->Core.enmType)
2157 {
2158 case RTR0MEMOBJTYPE_CONT:
2159 return pMemLnx->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);
2160
2161 case RTR0MEMOBJTYPE_PHYS:
2162 return pMemLnx->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);
2163
2164 /* the parent knows */
2165 case RTR0MEMOBJTYPE_MAPPING:
2166 return rtR0MemObjNativeGetPagePhysAddr(pMemLnx->Core.uRel.Child.pParent, iPage);
2167
2168 /* cPages > 0 */
2169 case RTR0MEMOBJTYPE_LOW:
2170 case RTR0MEMOBJTYPE_LOCK:
2171 case RTR0MEMOBJTYPE_PHYS_NC:
2172 case RTR0MEMOBJTYPE_PAGE:
2173 case RTR0MEMOBJTYPE_LARGE_PAGE:
2174 default:
2175 AssertMsgFailed(("%d\n", pMemLnx->Core.enmType));
2176 RT_FALL_THROUGH();
2177
2178 case RTR0MEMOBJTYPE_RES_VIRT:
2179 return NIL_RTHCPHYS;
2180 }
2181}
2182
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use