memobj-r0drv-linux.c@ 85698

Last change on this file since 85698 was 85698, checked in by vboxsync, 5 years ago
IPRT,lnx-kmods: Use new linux kernel version checking macros. Moved them to separate wrapper header.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Rev Revision`
File size: 63.3 KB

Line
1	/* $Id: memobj-r0drv-linux.c 85698 2020-08-11 17:05:29Z vboxsync $ */
2	/** @file
3	* IPRT - Ring-0 Memory Objects, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "the-linux-kernel.h"
32
33	#include <iprt/memobj.h>
34	#include <iprt/assert.h>
35	#include <iprt/err.h>
36	#include <iprt/log.h>
37	#include <iprt/mem.h>
38	#include <iprt/process.h>
39	#include <iprt/string.h>
40	#include "internal/memobj.h"
41	#include "internal/iprt.h"
42
43
44	/*********************************************************************************************************************************
45	* Defined Constants And Macros *
46	*********************************************************************************************************************************/
47	/* early 2.6 kernels */
48	#ifndef PAGE_SHARED_EXEC
49	# define PAGE_SHARED_EXEC PAGE_SHARED
50	#endif
51	#ifndef PAGE_READONLY_EXEC
52	# define PAGE_READONLY_EXEC PAGE_READONLY
53	#endif
54
55	/** @def IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
56	* Whether we use alloc_vm_area (3.2+) for executable memory.
57	* This is a must for 5.8+, but we enable it all the way back to 3.2.x for
58	* better W^R compliance (fExecutable flag). */
59	#if RTLNX_VER_MIN(3,2,0) \|\| defined(DOXYGEN_RUNNING)
60	# define IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
61	#endif
62
63	/*
64	* 2.6.29+ kernels don't work with remap_pfn_range() anymore because
65	* track_pfn_vma_new() is apparently not defined for non-RAM pages.
66	* It should be safe to use vm_insert_page() older kernels as well.
67	*/
68	#if RTLNX_VER_MIN(2,6,23)
69	# define VBOX_USE_INSERT_PAGE
70	#endif
71	#if defined(CONFIG_X86_PAE) \
72	&& ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
73	\|\| RTLNX_VER_RANGE(2,6,0, 2,6,11) )
74	# define VBOX_USE_PAE_HACK
75	#endif
76
77	/* gfp_t was introduced in 2.6.14, define it for earlier. */
78	#if RTLNX_VER_MAX(2,6,14)
79	# define gfp_t unsigned
80	#endif
81
82	/*
83	* Wrappers around mmap_lock/mmap_sem difference.
84	*/
85	#if RTLNX_VER_MIN(5,8,0)
86	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_lock)
87	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_lock)
88	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_lock)
89	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_lock)
90	#else
91	# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_sem)
92	# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_sem)
93	# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_sem)
94	# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_sem)
95	#endif
96
97
98	/*********************************************************************************************************************************
99	* Structures and Typedefs *
100	*********************************************************************************************************************************/
101	/**
102	* The Linux version of the memory object structure.
103	*/
104	typedef struct RTR0MEMOBJLNX
105	{
106	/** The core structure. */
107	RTR0MEMOBJINTERNAL Core;
108	/** Set if the allocation is contiguous.
109	* This means it has to be given back as one chunk. */
110	bool fContiguous;
111	/** Set if executable allocation. */
112	bool fExecutable;
113	/** Set if we've vmap'ed the memory into ring-0. */
114	bool fMappedToRing0;
115	#ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
116	/** Return from alloc_vm_area() that we now need to use for executable
117	* memory. */
118	struct vm_struct *pArea;
119	/** PTE array that goes along with pArea (must be freed). */
120	pte_t **papPtesForArea;
121	#endif
122	/** The pages in the apPages array. */
123	size_t cPages;
124	/** Array of struct page pointers. (variable size) */
125	struct page *apPages[1];
126	} RTR0MEMOBJLNX;
127	/** Pointer to the linux memory object. */
128	typedef RTR0MEMOBJLNX *PRTR0MEMOBJLNX;
129
130
131	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx);
132
133
134	/**
135	* Helper that converts from a RTR0PROCESS handle to a linux task.
136	*
137	* @returns The corresponding Linux task.
138	* @param R0Process IPRT ring-0 process handle.
139	*/
140	static struct task_struct *rtR0ProcessToLinuxTask(RTR0PROCESS R0Process)
141	{
142	/** @todo fix rtR0ProcessToLinuxTask!! */
143	/** @todo many (all?) callers currently assume that we return 'current'! */
144	return R0Process == RTR0ProcHandleSelf() ? current : NULL;
145	}
146
147
148	/**
149	* Compute order. Some functions allocate 2^order pages.
150	*
151	* @returns order.
152	* @param cPages Number of pages.
153	*/
154	static int rtR0MemObjLinuxOrder(size_t cPages)
155	{
156	int iOrder;
157	size_t cTmp;
158
159	for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
160	;
161	if (cPages & ~((size_t)1 << iOrder))
162	++iOrder;
163
164	return iOrder;
165	}
166
167
168	/**
169	* Converts from RTMEM_PROT_* to Linux PAGE_*.
170	*
171	* @returns Linux page protection constant.
172	* @param fProt The IPRT protection mask.
173	* @param fKernel Whether it applies to kernel or user space.
174	*/
175	static pgprot_t rtR0MemObjLinuxConvertProt(unsigned fProt, bool fKernel)
176	{
177	switch (fProt)
178	{
179	default:
180	AssertMsgFailed(("%#x %d\n", fProt, fKernel)); RT_FALL_THRU();
181	case RTMEM_PROT_NONE:
182	return PAGE_NONE;
183
184	case RTMEM_PROT_READ:
185	return fKernel ? PAGE_KERNEL_RO : PAGE_READONLY;
186
187	case RTMEM_PROT_WRITE:
188	case RTMEM_PROT_WRITE \| RTMEM_PROT_READ:
189	return fKernel ? PAGE_KERNEL : PAGE_SHARED;
190
191	case RTMEM_PROT_EXEC:
192	case RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
193	#if defined(RT_ARCH_X86) \|\| defined(RT_ARCH_AMD64)
194	if (fKernel)
195	{
196	pgprot_t fPg = MY_PAGE_KERNEL_EXEC;
197	pgprot_val(fPg) &= ~_PAGE_RW;
198	return fPg;
199	}
200	return PAGE_READONLY_EXEC;
201	#else
202	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_READONLY_EXEC;
203	#endif
204
205	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC:
206	case RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC \| RTMEM_PROT_READ:
207	return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_SHARED_EXEC;
208	}
209	}
210
211
212	/**
213	* Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
214	* an empty user space mapping.
215	*
216	* We acquire the mmap_sem/mmap_lock of the task!
217	*
218	* @returns Pointer to the mapping.
219	* (void *)-1 on failure.
220	* @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
221	* @param cb The size of the mapping.
222	* @param uAlignment The alignment of the mapping.
223	* @param pTask The Linux task to create this mapping in.
224	* @param fProt The RTMEM_PROT_* mask.
225	*/
226	static void rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, struct task_struct pTask, unsigned fProt)
227	{
228	unsigned fLnxProt;
229	unsigned long ulAddr;
230
231	Assert(pTask == current); /* do_mmap */
232	RT_NOREF_PV(pTask);
233
234	/*
235	* Convert from IPRT protection to mman.h PROT_ and call do_mmap.
236	*/
237	fProt &= (RTMEM_PROT_NONE \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC);
238	if (fProt == RTMEM_PROT_NONE)
239	fLnxProt = PROT_NONE;
240	else
241	{
242	fLnxProt = 0;
243	if (fProt & RTMEM_PROT_READ)
244	fLnxProt \|= PROT_READ;
245	if (fProt & RTMEM_PROT_WRITE)
246	fLnxProt \|= PROT_WRITE;
247	if (fProt & RTMEM_PROT_EXEC)
248	fLnxProt \|= PROT_EXEC;
249	}
250
251	if (R3PtrFixed != (RTR3PTR)-1)
252	{
253	#if RTLNX_VER_MIN(3,5,0)
254	ulAddr = vm_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
255	#else
256	LNX_MM_DOWN_WRITE(pTask->mm);
257	ulAddr = do_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS \| MAP_FIXED, 0);
258	LNX_MM_UP_WRITE(pTask->mm);
259	#endif
260	}
261	else
262	{
263	#if RTLNX_VER_MIN(3,5,0)
264	ulAddr = vm_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
265	#else
266	LNX_MM_DOWN_WRITE(pTask->mm);
267	ulAddr = do_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED \| MAP_ANONYMOUS, 0);
268	LNX_MM_UP_WRITE(pTask->mm);
269	#endif
270	if ( !(ulAddr & ~PAGE_MASK)
271	&& (ulAddr & (uAlignment - 1)))
272	{
273	/** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
274	* up alignment gaps. This is of course complicated by fragmentation (which we might have cause
275	* ourselves) and further by there begin two mmap strategies (top / bottom). */
276	/* For now, just ignore uAlignment requirements... */
277	}
278	}
279
280
281	if (ulAddr & ~PAGE_MASK) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
282	return (void *)-1;
283	return (void *)ulAddr;
284	}
285
286
287	/**
288	* Worker that destroys a user space mapping.
289	* Undoes what rtR0MemObjLinuxDoMmap did.
290	*
291	* We acquire the mmap_sem/mmap_lock of the task!
292	*
293	* @param pv The ring-3 mapping.
294	* @param cb The size of the mapping.
295	* @param pTask The Linux task to destroy this mapping in.
296	*/
297	static void rtR0MemObjLinuxDoMunmap(void pv, size_t cb, struct task_struct pTask)
298	{
299	#if RTLNX_VER_MIN(3,5,0)
300	Assert(pTask == current); RT_NOREF_PV(pTask);
301	vm_munmap((unsigned long)pv, cb);
302	#elif defined(USE_RHEL4_MUNMAP)
303	LNX_MM_DOWN_WRITE(pTask->mm);
304	do_munmap(pTask->mm, (unsigned long)pv, cb, 0); /* should it be 1 or 0? */
305	LNX_MM_UP_WRITE(pTask->mm);
306	#else
307	LNX_MM_DOWN_WRITE(pTask->mm);
308	do_munmap(pTask->mm, (unsigned long)pv, cb);
309	LNX_MM_UP_WRITE(pTask->mm);
310	#endif
311	}
312
313
314	/**
315	* Internal worker that allocates physical pages and creates the memory object for them.
316	*
317	* @returns IPRT status code.
318	* @param ppMemLnx Where to store the memory object pointer.
319	* @param enmType The object type.
320	* @param cb The number of bytes to allocate.
321	* @param uAlignment The alignment of the physical memory.
322	* Only valid if fContiguous == true, ignored otherwise.
323	* @param fFlagsLnx The page allocation flags (GPFs).
324	* @param fContiguous Whether the allocation must be contiguous.
325	* @param fExecutable Whether the memory must be executable.
326	* @param rcNoMem What to return when we're out of pages.
327	*/
328	static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX *ppMemLnx, RTR0MEMOBJTYPE enmType, size_t cb,
329	size_t uAlignment, gfp_t fFlagsLnx, bool fContiguous, bool fExecutable, int rcNoMem)
330	{
331	size_t iPage;
332	size_t const cPages = cb >> PAGE_SHIFT;
333	struct page *paPages;
334
335	/*
336	* Allocate a memory object structure that's large enough to contain
337	* the page pointer array.
338	*/
339	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), enmType, NULL, cb);
340	if (!pMemLnx)
341	return VERR_NO_MEMORY;
342	pMemLnx->cPages = cPages;
343
344	if (cPages > 255)
345	{
346	# ifdef __GFP_REPEAT
347	/* Try hard to allocate the memory, but the allocation attempt might fail. */
348	fFlagsLnx \|= __GFP_REPEAT;
349	# endif
350	# ifdef __GFP_NOMEMALLOC
351	/* Introduced with Linux 2.6.12: Don't use emergency reserves */
352	fFlagsLnx \|= __GFP_NOMEMALLOC;
353	# endif
354	}
355
356	/*
357	* Allocate the pages.
358	* For small allocations we'll try contiguous first and then fall back on page by page.
359	*/
360	#if RTLNX_VER_MIN(2,4,22)
361	if ( fContiguous
362	\|\| cb <= PAGE_SIZE * 2)
363	{
364	# ifdef VBOX_USE_INSERT_PAGE
365	paPages = alloc_pages(fFlagsLnx \| __GFP_COMP \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
366	# else
367	paPages = alloc_pages(fFlagsLnx \| __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
368	# endif
369	if (paPages)
370	{
371	fContiguous = true;
372	for (iPage = 0; iPage < cPages; iPage++)
373	pMemLnx->apPages[iPage] = &paPages[iPage];
374	}
375	else if (fContiguous)
376	{
377	rtR0MemObjDelete(&pMemLnx->Core);
378	return rcNoMem;
379	}
380	}
381
382	if (!fContiguous)
383	{
384	for (iPage = 0; iPage < cPages; iPage++)
385	{
386	pMemLnx->apPages[iPage] = alloc_page(fFlagsLnx \| __GFP_NOWARN);
387	if (RT_UNLIKELY(!pMemLnx->apPages[iPage]))
388	{
389	while (iPage-- > 0)
390	__free_page(pMemLnx->apPages[iPage]);
391	rtR0MemObjDelete(&pMemLnx->Core);
392	return rcNoMem;
393	}
394	}
395	}
396
397	#else /* < 2.4.22 */
398	/** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
399	paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
400	if (!paPages)
401	{
402	rtR0MemObjDelete(&pMemLnx->Core);
403	return rcNoMem;
404	}
405	for (iPage = 0; iPage < cPages; iPage++)
406	{
407	pMemLnx->apPages[iPage] = &paPages[iPage];
408	if (fExecutable)
409	MY_SET_PAGES_EXEC(pMemLnx->apPages[iPage], 1);
410	if (PageHighMem(pMemLnx->apPages[iPage]))
411	BUG();
412	}
413
414	fContiguous = true;
415	#endif /* < 2.4.22 */
416	pMemLnx->fContiguous = fContiguous;
417	pMemLnx->fExecutable = fExecutable;
418
419	#if RTLNX_VER_MAX(4,5,0)
420	/*
421	* Reserve the pages.
422	*
423	* Linux >= 4.5 with CONFIG_DEBUG_VM panics when setting PG_reserved on compound
424	* pages. According to Michal Hocko this shouldn't be necessary anyway because
425	* as pages which are not on the LRU list are never evictable.
426	*/
427	for (iPage = 0; iPage < cPages; iPage++)
428	SetPageReserved(pMemLnx->apPages[iPage]);
429	#endif
430
431	/*
432	* Note that the physical address of memory allocated with alloc_pages(flags, order)
433	* is always 2^(PAGE_SHIFT+order)-aligned.
434	*/
435	if ( fContiguous
436	&& uAlignment > PAGE_SIZE)
437	{
438	/*
439	* Check for alignment constraints. The physical address of memory allocated with
440	* alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
441	*/
442	if (RT_UNLIKELY(page_to_phys(pMemLnx->apPages[0]) & (uAlignment - 1)))
443	{
444	/*
445	* This should never happen!
446	*/
447	printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
448	(unsigned long)cb, (unsigned long)uAlignment, rtR0MemObjLinuxOrder(cPages), (unsigned long)page_to_phys(pMemLnx->apPages[0]));
449	rtR0MemObjLinuxFreePages(pMemLnx);
450	return rcNoMem;
451	}
452	}
453
454	*ppMemLnx = pMemLnx;
455	return VINF_SUCCESS;
456	}
457
458
459	/**
460	* Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
461	*
462	* This method does NOT free the object.
463	*
464	* @param pMemLnx The object which physical pages should be freed.
465	*/
466	static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx)
467	{
468	size_t iPage = pMemLnx->cPages;
469	if (iPage > 0)
470	{
471	/*
472	* Restore the page flags.
473	*/
474	while (iPage-- > 0)
475	{
476	#if RTLNX_VER_MAX(4,5,0)
477	/* See SetPageReserved() in rtR0MemObjLinuxAllocPages() */
478	ClearPageReserved(pMemLnx->apPages[iPage]);
479	#endif
480	#if RTLNX_VER_MAX(2,4,22)
481	if (pMemLnx->fExecutable)
482	MY_SET_PAGES_NOEXEC(pMemLnx->apPages[iPage], 1);
483	#endif
484	}
485
486	/*
487	* Free the pages.
488	*/
489	#if RTLNX_VER_MIN(2,4,22)
490	if (!pMemLnx->fContiguous)
491	{
492	iPage = pMemLnx->cPages;
493	while (iPage-- > 0)
494	__free_page(pMemLnx->apPages[iPage]);
495	}
496	else
497	#endif
498	__free_pages(pMemLnx->apPages[0], rtR0MemObjLinuxOrder(pMemLnx->cPages));
499
500	pMemLnx->cPages = 0;
501	}
502	}
503
504
505	/**
506	* Maps the allocation into ring-0.
507	*
508	* This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
509	*
510	* Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
511	* space, so we'll use that mapping if possible. If execute access is required, we'll
512	* play safe and do our own mapping.
513	*
514	* @returns IPRT status code.
515	* @param pMemLnx The linux memory object to map.
516	* @param fExecutable Whether execute access is required.
517	*/
518	static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx, bool fExecutable)
519	{
520	int rc = VINF_SUCCESS;
521
522	/*
523	* Choose mapping strategy.
524	*/
525	bool fMustMap = fExecutable
526	\|\| !pMemLnx->fContiguous;
527	if (!fMustMap)
528	{
529	size_t iPage = pMemLnx->cPages;
530	while (iPage-- > 0)
531	if (PageHighMem(pMemLnx->apPages[iPage]))
532	{
533	fMustMap = true;
534	break;
535	}
536	}
537
538	Assert(!pMemLnx->Core.pv);
539	Assert(!pMemLnx->fMappedToRing0);
540
541	if (fMustMap)
542	{
543	/*
544	* Use vmap - 2.4.22 and later.
545	*/
546	#if RTLNX_VER_MIN(2,4,22)
547	pgprot_t fPg;
548	pgprot_val(fPg) = _PAGE_PRESENT \| _PAGE_RW;
549	# ifdef _PAGE_NX
550	if (!fExecutable)
551	pgprot_val(fPg) \|= _PAGE_NX;
552	# endif
553
554	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
555	if (fExecutable)
556	{
557	pte_t papPtes = (pte_t )kmalloc_array(pMemLnx->cPages, sizeof(papPtes[0]), GFP_KERNEL);
558	if (papPtes)
559	{
560	pMemLnx->pArea = alloc_vm_area(pMemLnx->Core.cb, papPtes); /* Note! pArea->nr_pages is not set. */
561	if (pMemLnx->pArea)
562	{
563	size_t i;
564	Assert(pMemLnx->pArea->size >= pMemLnx->Core.cb); /* Note! includes guard page. */
565	Assert(pMemLnx->pArea->addr);
566	# ifdef _PAGE_NX
567	pgprot_val(fPg) \|= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
568	# endif
569	pMemLnx->papPtesForArea = papPtes;
570	for (i = 0; i < pMemLnx->cPages; i++)
571	*papPtes[i] = mk_pte(pMemLnx->apPages[i], fPg);
572	pMemLnx->Core.pv = pMemLnx->pArea->addr;
573	pMemLnx->fMappedToRing0 = true;
574	}
575	else
576	{
577	kfree(papPtes);
578	rc = VERR_MAP_FAILED;
579	}
580	}
581	else
582	rc = VERR_MAP_FAILED;
583	}
584	else
585	# endif
586	{
587	# ifdef VM_MAP
588	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_MAP, fPg);
589	# else
590	pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_ALLOC, fPg);
591	# endif
592	if (pMemLnx->Core.pv)
593	pMemLnx->fMappedToRing0 = true;
594	else
595	rc = VERR_MAP_FAILED;
596	}
597	#else /* < 2.4.22 */
598	rc = VERR_NOT_SUPPORTED;
599	#endif
600	}
601	else
602	{
603	/*
604	* Use the kernel RAM mapping.
605	*/
606	pMemLnx->Core.pv = phys_to_virt(page_to_phys(pMemLnx->apPages[0]));
607	Assert(pMemLnx->Core.pv);
608	}
609
610	return rc;
611	}
612
613
614	/**
615	* Undoes what rtR0MemObjLinuxVMap() did.
616	*
617	* @param pMemLnx The linux memory object.
618	*/
619	static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx)
620	{
621	#if RTLNX_VER_MIN(2,4,22)
622	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
623	if (pMemLnx->pArea)
624	{
625	# if 0
626	pte_t **papPtes = pMemLnx->papPtesForArea;
627	size_t i;
628	for (i = 0; i < pMemLnx->cPages; i++)
629	*papPtes[i] = 0;
630	# endif
631	free_vm_area(pMemLnx->pArea);
632	kfree(pMemLnx->papPtesForArea);
633	pMemLnx->pArea = NULL;
634	pMemLnx->papPtesForArea = NULL;
635	}
636	else
637	# endif
638	if (pMemLnx->fMappedToRing0)
639	{
640	Assert(pMemLnx->Core.pv);
641	vunmap(pMemLnx->Core.pv);
642	pMemLnx->fMappedToRing0 = false;
643	}
644	#else /* < 2.4.22 */
645	Assert(!pMemLnx->fMappedToRing0);
646	#endif
647	pMemLnx->Core.pv = NULL;
648	}
649
650
651	DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
652	{
653	IPRT_LINUX_SAVE_EFL_AC();
654	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
655
656	/*
657	* Release any memory that we've allocated or locked.
658	*/
659	switch (pMemLnx->Core.enmType)
660	{
661	case RTR0MEMOBJTYPE_LOW:
662	case RTR0MEMOBJTYPE_PAGE:
663	case RTR0MEMOBJTYPE_CONT:
664	case RTR0MEMOBJTYPE_PHYS:
665	case RTR0MEMOBJTYPE_PHYS_NC:
666	rtR0MemObjLinuxVUnmap(pMemLnx);
667	rtR0MemObjLinuxFreePages(pMemLnx);
668	break;
669
670	case RTR0MEMOBJTYPE_LOCK:
671	if (pMemLnx->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
672	{
673	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
674	size_t iPage;
675	Assert(pTask);
676	if (pTask && pTask->mm)
677	LNX_MM_DOWN_READ(pTask->mm);
678
679	iPage = pMemLnx->cPages;
680	while (iPage-- > 0)
681	{
682	if (!PageReserved(pMemLnx->apPages[iPage]))
683	SetPageDirty(pMemLnx->apPages[iPage]);
684	#if RTLNX_VER_MIN(4,6,0)
685	put_page(pMemLnx->apPages[iPage]);
686	#else
687	page_cache_release(pMemLnx->apPages[iPage]);
688	#endif
689	}
690
691	if (pTask && pTask->mm)
692	LNX_MM_UP_READ(pTask->mm);
693	}
694	/* else: kernel memory - nothing to do here. */
695	break;
696
697	case RTR0MEMOBJTYPE_RES_VIRT:
698	Assert(pMemLnx->Core.pv);
699	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
700	{
701	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
702	Assert(pTask);
703	if (pTask && pTask->mm)
704	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
705	}
706	else
707	{
708	vunmap(pMemLnx->Core.pv);
709
710	Assert(pMemLnx->cPages == 1 && pMemLnx->apPages[0] != NULL);
711	__free_page(pMemLnx->apPages[0]);
712	pMemLnx->apPages[0] = NULL;
713	pMemLnx->cPages = 0;
714	}
715	pMemLnx->Core.pv = NULL;
716	break;
717
718	case RTR0MEMOBJTYPE_MAPPING:
719	Assert(pMemLnx->cPages == 0); Assert(pMemLnx->Core.pv);
720	if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
721	{
722	struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
723	Assert(pTask);
724	if (pTask && pTask->mm)
725	rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
726	}
727	else
728	vunmap(pMemLnx->Core.pv);
729	pMemLnx->Core.pv = NULL;
730	break;
731
732	default:
733	AssertMsgFailed(("enmType=%d\n", pMemLnx->Core.enmType));
734	return VERR_INTERNAL_ERROR;
735	}
736	IPRT_LINUX_RESTORE_EFL_ONLY_AC();
737	return VINF_SUCCESS;
738	}
739
740
741	DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
742	{
743	IPRT_LINUX_SAVE_EFL_AC();
744	PRTR0MEMOBJLNX pMemLnx;
745	int rc;
746
747	#if RTLNX_VER_MIN(2,4,22)
748	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_HIGHUSER,
749	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
750	#else
751	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_USER,
752	false /* non-contiguous */, fExecutable, VERR_NO_MEMORY);
753	#endif
754	if (RT_SUCCESS(rc))
755	{
756	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
757	if (RT_SUCCESS(rc))
758	{
759	*ppMem = &pMemLnx->Core;
760	IPRT_LINUX_RESTORE_EFL_AC();
761	return rc;
762	}
763
764	rtR0MemObjLinuxFreePages(pMemLnx);
765	rtR0MemObjDelete(&pMemLnx->Core);
766	}
767
768	IPRT_LINUX_RESTORE_EFL_AC();
769	return rc;
770	}
771
772
773	DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
774	{
775	IPRT_LINUX_SAVE_EFL_AC();
776	PRTR0MEMOBJLNX pMemLnx;
777	int rc;
778
779	/* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
780	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
781	/* ZONE_DMA32: 0-4GB */
782	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA32,
783	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
784	if (RT_FAILURE(rc))
785	#endif
786	#ifdef RT_ARCH_AMD64
787	/* ZONE_DMA: 0-16MB */
788	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA,
789	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
790	#else
791	# ifdef CONFIG_X86_PAE
792	# endif
793	/* ZONE_NORMAL: 0-896MB */
794	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_USER,
795	false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY);
796	#endif
797	if (RT_SUCCESS(rc))
798	{
799	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
800	if (RT_SUCCESS(rc))
801	{
802	*ppMem = &pMemLnx->Core;
803	IPRT_LINUX_RESTORE_EFL_AC();
804	return rc;
805	}
806
807	rtR0MemObjLinuxFreePages(pMemLnx);
808	rtR0MemObjDelete(&pMemLnx->Core);
809	}
810
811	IPRT_LINUX_RESTORE_EFL_AC();
812	return rc;
813	}
814
815
816	DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
817	{
818	IPRT_LINUX_SAVE_EFL_AC();
819	PRTR0MEMOBJLNX pMemLnx;
820	int rc;
821
822	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
823	/* ZONE_DMA32: 0-4GB */
824	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA32,
825	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
826	if (RT_FAILURE(rc))
827	#endif
828	#ifdef RT_ARCH_AMD64
829	/* ZONE_DMA: 0-16MB */
830	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA,
831	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
832	#else
833	/* ZONE_NORMAL (32-bit hosts): 0-896MB */
834	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_USER,
835	true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY);
836	#endif
837	if (RT_SUCCESS(rc))
838	{
839	rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
840	if (RT_SUCCESS(rc))
841	{
842	#if defined(RT_STRICT) && (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_HIGHMEM64G))
843	size_t iPage = pMemLnx->cPages;
844	while (iPage-- > 0)
845	Assert(page_to_phys(pMemLnx->apPages[iPage]) < _4G);
846	#endif
847	pMemLnx->Core.u.Cont.Phys = page_to_phys(pMemLnx->apPages[0]);
848	*ppMem = &pMemLnx->Core;
849	IPRT_LINUX_RESTORE_EFL_AC();
850	return rc;
851	}
852
853	rtR0MemObjLinuxFreePages(pMemLnx);
854	rtR0MemObjDelete(&pMemLnx->Core);
855	}
856
857	IPRT_LINUX_RESTORE_EFL_AC();
858	return rc;
859	}
860
861
862	/**
863	* Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
864	*
865	* @returns IPRT status code.
866	* @param ppMemLnx Where to
867	* @param enmType The object type.
868	* @param cb The size of the allocation.
869	* @param uAlignment The alignment of the physical memory.
870	* Only valid for fContiguous == true, ignored otherwise.
871	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
872	* @param fGfp The Linux GFP flags to use for the allocation.
873	*/
874	static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
875	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, gfp_t fGfp)
876	{
877	PRTR0MEMOBJLNX pMemLnx;
878	int rc;
879
880	rc = rtR0MemObjLinuxAllocPages(&pMemLnx, enmType, cb, uAlignment, fGfp,
881	enmType == RTR0MEMOBJTYPE_PHYS /* contiguous / non-contiguous */,
882	false /fExecutable/, VERR_NO_PHYS_MEMORY);
883	if (RT_FAILURE(rc))
884	return rc;
885
886	/*
887	* Check the addresses if necessary. (Can be optimized a bit for PHYS.)
888	*/
889	if (PhysHighest != NIL_RTHCPHYS)
890	{
891	size_t iPage = pMemLnx->cPages;
892	while (iPage-- > 0)
893	if (page_to_phys(pMemLnx->apPages[iPage]) > PhysHighest)
894	{
895	rtR0MemObjLinuxFreePages(pMemLnx);
896	rtR0MemObjDelete(&pMemLnx->Core);
897	return VERR_NO_MEMORY;
898	}
899	}
900
901	/*
902	* Complete the object.
903	*/
904	if (enmType == RTR0MEMOBJTYPE_PHYS)
905	{
906	pMemLnx->Core.u.Phys.PhysBase = page_to_phys(pMemLnx->apPages[0]);
907	pMemLnx->Core.u.Phys.fAllocated = true;
908	}
909	*ppMem = &pMemLnx->Core;
910	return rc;
911	}
912
913
914	/**
915	* Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
916	*
917	* @returns IPRT status code.
918	* @param ppMem Where to store the memory object pointer on success.
919	* @param enmType The object type.
920	* @param cb The size of the allocation.
921	* @param uAlignment The alignment of the physical memory.
922	* Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
923	* @param PhysHighest See rtR0MemObjNativeAllocPhys.
924	*/
925	static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
926	size_t cb, size_t uAlignment, RTHCPHYS PhysHighest)
927	{
928	int rc;
929	IPRT_LINUX_SAVE_EFL_AC();
930
931	/*
932	* There are two clear cases and that's the <=16MB and anything-goes ones.
933	* When the physical address limit is somewhere in-between those two we'll
934	* just have to try, starting with HIGHUSER and working our way thru the
935	* different types, hoping we'll get lucky.
936	*
937	* We should probably move this physical address restriction logic up to
938	* the page alloc function as it would be more efficient there. But since
939	* we don't expect this to be a performance issue just yet it can wait.
940	*/
941	if (PhysHighest == NIL_RTHCPHYS)
942	/* ZONE_HIGHMEM: the whole physical memory */
943	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
944	else if (PhysHighest <= _1M * 16)
945	/* ZONE_DMA: 0-16MB */
946	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
947	else
948	{
949	rc = VERR_NO_MEMORY;
950	if (RT_FAILURE(rc))
951	/* ZONE_HIGHMEM: the whole physical memory */
952	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
953	if (RT_FAILURE(rc))
954	/* ZONE_NORMAL: 0-896MB */
955	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_USER);
956	#ifdef GFP_DMA32
957	if (RT_FAILURE(rc))
958	/* ZONE_DMA32: 0-4GB */
959	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA32);
960	#endif
961	if (RT_FAILURE(rc))
962	/* ZONE_DMA: 0-16MB */
963	rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
964	}
965	IPRT_LINUX_RESTORE_EFL_AC();
966	return rc;
967	}
968
969
970	/**
971	* Translates a kernel virtual address to a linux page structure by walking the
972	* page tables.
973	*
974	* @note We do assume that the page tables will not change as we are walking
975	* them. This assumption is rather forced by the fact that I could not
976	* immediately see any way of preventing this from happening. So, we
977	* take some extra care when accessing them.
978	*
979	* Because of this, we don't want to use this function on memory where
980	* attribute changes to nearby pages is likely to cause large pages to
981	* be used or split up. So, don't use this for the linear mapping of
982	* physical memory.
983	*
984	* @returns Pointer to the page structur or NULL if it could not be found.
985	* @param pv The kernel virtual address.
986	*/
987	RTDECL(struct page ) rtR0MemObjLinuxVirtToPage(void pv)
988	{
989	unsigned long ulAddr = (unsigned long)pv;
990	unsigned long pfn;
991	struct page *pPage;
992	pte_t *pEntry;
993	union
994	{
995	pgd_t Global;
996	#if RTLNX_VER_MIN(4,12,0)
997	p4d_t Four;
998	#endif
999	#if RTLNX_VER_MIN(2,6,11)
1000	pud_t Upper;
1001	#endif
1002	pmd_t Middle;
1003	pte_t Entry;
1004	} u;
1005
1006	/* Should this happen in a situation this code will be called in? And if
1007	* so, can it change under our feet? See also
1008	* "Documentation/vm/active_mm.txt" in the kernel sources. */
1009	if (RT_UNLIKELY(!current->active_mm))
1010	return NULL;
1011	u.Global = *pgd_offset(current->active_mm, ulAddr);
1012	if (RT_UNLIKELY(pgd_none(u.Global)))
1013	return NULL;
1014	#if RTLNX_VER_MIN(2,6,11)
1015	# if RTLNX_VER_MIN(4,12,0)
1016	u.Four = *p4d_offset(&u.Global, ulAddr);
1017	if (RT_UNLIKELY(p4d_none(u.Four)))
1018	return NULL;
1019	if (p4d_large(u.Four))
1020	{
1021	pPage = p4d_page(u.Four);
1022	AssertReturn(pPage, NULL);
1023	pfn = page_to_pfn(pPage); /* doing the safe way... */
1024	AssertCompile(P4D_SHIFT - PAGE_SHIFT < 31);
1025	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (P4D_SHIFT - PAGE_SHIFT)) - 1);
1026	return pfn_to_page(pfn);
1027	}
1028	u.Upper = *pud_offset(&u.Four, ulAddr);
1029	# else /* < 4.12 */
1030	u.Upper = *pud_offset(&u.Global, ulAddr);
1031	# endif /* < 4.12 */
1032	if (RT_UNLIKELY(pud_none(u.Upper)))
1033	return NULL;
1034	# if RTLNX_VER_MIN(2,6,25)
1035	if (pud_large(u.Upper))
1036	{
1037	pPage = pud_page(u.Upper);
1038	AssertReturn(pPage, NULL);
1039	pfn = page_to_pfn(pPage); /* doing the safe way... */
1040	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PUD_SHIFT - PAGE_SHIFT)) - 1);
1041	return pfn_to_page(pfn);
1042	}
1043	# endif
1044	u.Middle = *pmd_offset(&u.Upper, ulAddr);
1045	#else /* < 2.6.11 */
1046	u.Middle = *pmd_offset(&u.Global, ulAddr);
1047	#endif /* < 2.6.11 */
1048	if (RT_UNLIKELY(pmd_none(u.Middle)))
1049	return NULL;
1050	#if RTLNX_VER_MIN(2,6,0)
1051	if (pmd_large(u.Middle))
1052	{
1053	pPage = pmd_page(u.Middle);
1054	AssertReturn(pPage, NULL);
1055	pfn = page_to_pfn(pPage); /* doing the safe way... */
1056	pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PMD_SHIFT - PAGE_SHIFT)) - 1);
1057	return pfn_to_page(pfn);
1058	}
1059	#endif
1060
1061	#if RTLNX_VER_MIN(2,5,5) \|\| defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
1062	pEntry = pte_offset_map(&u.Middle, ulAddr);
1063	#else
1064	pEntry = pte_offset(&u.Middle, ulAddr);
1065	#endif
1066	if (RT_UNLIKELY(!pEntry))
1067	return NULL;
1068	u.Entry = *pEntry;
1069	#if RTLNX_VER_MIN(2,5,5) \|\| defined(pte_offset_map)
1070	pte_unmap(pEntry);
1071	#endif
1072
1073	if (RT_UNLIKELY(!pte_present(u.Entry)))
1074	return NULL;
1075	return pte_page(u.Entry);
1076	}
1077	RT_EXPORT_SYMBOL(rtR0MemObjLinuxVirtToPage);
1078
1079
1080	DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment)
1081	{
1082	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS, cb, uAlignment, PhysHighest);
1083	}
1084
1085
1086	DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest)
1087	{
1088	return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS_NC, cb, PAGE_SIZE, PhysHighest);
1089	}
1090
1091
1092	DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy)
1093	{
1094	IPRT_LINUX_SAVE_EFL_AC();
1095
1096	/*
1097	* All we need to do here is to validate that we can use
1098	* ioremap on the specified address (32/64-bit dma_addr_t).
1099	*/
1100	PRTR0MEMOBJLNX pMemLnx;
1101	dma_addr_t PhysAddr = Phys;
1102	AssertMsgReturn(PhysAddr == Phys, ("%#llx\n", (unsigned long long)Phys), VERR_ADDRESS_TOO_BIG);
1103
1104	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_PHYS, NULL, cb);
1105	if (!pMemLnx)
1106	{
1107	IPRT_LINUX_RESTORE_EFL_AC();
1108	return VERR_NO_MEMORY;
1109	}
1110
1111	pMemLnx->Core.u.Phys.PhysBase = PhysAddr;
1112	pMemLnx->Core.u.Phys.fAllocated = false;
1113	pMemLnx->Core.u.Phys.uCachePolicy = uCachePolicy;
1114	Assert(!pMemLnx->cPages);
1115	*ppMem = &pMemLnx->Core;
1116	IPRT_LINUX_RESTORE_EFL_AC();
1117	return VINF_SUCCESS;
1118	}
1119
1120	/* openSUSE Leap 42.3 detection :-/ */
1121	#if RTLNX_VER_RANGE(4,4,0, 4,6,0) && defined(FAULT_FLAG_REMOTE)
1122	# define GET_USER_PAGES_API KERNEL_VERSION(4, 10, 0) /* no typo! */
1123	#else
1124	# define GET_USER_PAGES_API LINUX_VERSION_CODE
1125	#endif
1126
1127	DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process)
1128	{
1129	IPRT_LINUX_SAVE_EFL_AC();
1130	const int cPages = cb >> PAGE_SHIFT;
1131	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1132	struct vm_area_struct **papVMAs;
1133	PRTR0MEMOBJLNX pMemLnx;
1134	int rc = VERR_NO_MEMORY;
1135	int const fWrite = fAccess & RTMEM_PROT_WRITE ? 1 : 0;
1136
1137	/*
1138	* Check for valid task and size overflows.
1139	*/
1140	if (!pTask)
1141	return VERR_NOT_SUPPORTED;
1142	if (((size_t)cPages << PAGE_SHIFT) != cb)
1143	return VERR_OUT_OF_RANGE;
1144
1145	/*
1146	* Allocate the memory object and a temporary buffer for the VMAs.
1147	*/
1148	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, (void *)R3Ptr, cb);
1149	if (!pMemLnx)
1150	{
1151	IPRT_LINUX_RESTORE_EFL_AC();
1152	return VERR_NO_MEMORY;
1153	}
1154
1155	papVMAs = (struct vm_area_struct *)RTMemAlloc(sizeof(papVMAs) * cPages);
1156	if (papVMAs)
1157	{
1158	LNX_MM_DOWN_READ(pTask->mm);
1159
1160	/*
1161	* Get user pages.
1162	*/
1163	/** @todo r=bird: Should we not force read access too? */
1164	#if GET_USER_PAGES_API >= KERNEL_VERSION(4, 6, 0)
1165	if (R0Process == RTR0ProcHandleSelf())
1166	rc = get_user_pages(R3Ptr, /* Where from. */
1167	cPages, /* How many pages. */
1168	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1169	fWrite ? FOLL_WRITE \| /* Write to memory. */
1170	FOLL_FORCE /* force write access. */
1171	: 0, /* Write to memory. */
1172	# else
1173	fWrite, /* Write to memory. */
1174	fWrite, /* force write access. */
1175	# endif
1176	&pMemLnx->apPages[0], /* Page array. */
1177	papVMAs); /* vmas */
1178	/*
1179	* Actually this should not happen at the moment as call this function
1180	* only for our own process.
1181	*/
1182	else
1183	rc = get_user_pages_remote(
1184	pTask, /* Task for fault accounting. */
1185	pTask->mm, /* Whose pages. */
1186	R3Ptr, /* Where from. */
1187	cPages, /* How many pages. */
1188	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
1189	fWrite ? FOLL_WRITE \| /* Write to memory. */
1190	FOLL_FORCE /* force write access. */
1191	: 0, /* Write to memory. */
1192	# else
1193	fWrite, /* Write to memory. */
1194	fWrite, /* force write access. */
1195	# endif
1196	&pMemLnx->apPages[0], /* Page array. */
1197	papVMAs /* vmas */
1198	# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 10, 0)
1199	, NULL /* locked */
1200	# endif
1201	);
1202	#else /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1203	rc = get_user_pages(pTask, /* Task for fault accounting. */
1204	pTask->mm, /* Whose pages. */
1205	R3Ptr, /* Where from. */
1206	cPages, /* How many pages. */
1207	/* The get_user_pages API change was back-ported to 4.4.168. */
1208	# if RTLNX_VER_RANGE(4,4,168, 4,5,0)
1209	fWrite ? FOLL_WRITE \| /* Write to memory. */
1210	FOLL_FORCE /* force write access. */
1211	: 0, /* Write to memory. */
1212	# else
1213	fWrite, /* Write to memory. */
1214	fWrite, /* force write access. */
1215	# endif
1216	&pMemLnx->apPages[0], /* Page array. */
1217	papVMAs); /* vmas */
1218	#endif /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
1219	if (rc == cPages)
1220	{
1221	/*
1222	* Flush dcache (required?), protect against fork and _really_ pin the page
1223	* table entries. get_user_pages() will protect against swapping out the
1224	* pages but it will NOT protect against removing page table entries. This
1225	* can be achieved with
1226	* - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
1227	* an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
1228	* Usual Linux distributions support only a limited size of locked pages
1229	* (e.g. 32KB).
1230	* - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
1231	* or by
1232	* - setting the VM_LOCKED flag. This is the same as doing mlock() without
1233	* a range check.
1234	*/
1235	/** @todo The Linux fork() protection will require more work if this API
1236	* is to be used for anything but locking VM pages. */
1237	while (rc-- > 0)
1238	{
1239	flush_dcache_page(pMemLnx->apPages[rc]);
1240	papVMAs[rc]->vm_flags \|= VM_DONTCOPY \| VM_LOCKED;
1241	}
1242
1243	LNX_MM_UP_READ(pTask->mm);
1244
1245	RTMemFree(papVMAs);
1246
1247	pMemLnx->Core.u.Lock.R0Process = R0Process;
1248	pMemLnx->cPages = cPages;
1249	Assert(!pMemLnx->fMappedToRing0);
1250	*ppMem = &pMemLnx->Core;
1251
1252	IPRT_LINUX_RESTORE_EFL_AC();
1253	return VINF_SUCCESS;
1254	}
1255
1256	/*
1257	* Failed - we need to unlock any pages that we succeeded to lock.
1258	*/
1259	while (rc-- > 0)
1260	{
1261	if (!PageReserved(pMemLnx->apPages[rc]))
1262	SetPageDirty(pMemLnx->apPages[rc]);
1263	#if RTLNX_VER_MIN(4,6,0)
1264	put_page(pMemLnx->apPages[rc]);
1265	#else
1266	page_cache_release(pMemLnx->apPages[rc]);
1267	#endif
1268	}
1269
1270	LNX_MM_UP_READ(pTask->mm);
1271
1272	RTMemFree(papVMAs);
1273	rc = VERR_LOCK_FAILED;
1274	}
1275
1276	rtR0MemObjDelete(&pMemLnx->Core);
1277	IPRT_LINUX_RESTORE_EFL_AC();
1278	return rc;
1279	}
1280
1281
1282	DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess)
1283	{
1284	IPRT_LINUX_SAVE_EFL_AC();
1285	void pvLast = (uint8_t )pv + cb - 1;
1286	size_t const cPages = cb >> PAGE_SHIFT;
1287	PRTR0MEMOBJLNX pMemLnx;
1288	bool fLinearMapping;
1289	int rc;
1290	uint8_t *pbPage;
1291	size_t iPage;
1292	NOREF(fAccess);
1293
1294	if ( !RTR0MemKernelIsValidAddr(pv)
1295	\|\| !RTR0MemKernelIsValidAddr(pv + cb))
1296	return VERR_INVALID_PARAMETER;
1297
1298	/*
1299	* The lower part of the kernel memory has a linear mapping between
1300	* physical and virtual addresses. So we take a short cut here. This is
1301	* assumed to be the cleanest way to handle those addresses (and the code
1302	* is well tested, though the test for determining it is not very nice).
1303	* If we ever decide it isn't we can still remove it.
1304	*/
1305	#if 0
1306	fLinearMapping = (unsigned long)pvLast < VMALLOC_START;
1307	#else
1308	fLinearMapping = (unsigned long)pv >= (unsigned long)__va(0)
1309	&& (unsigned long)pvLast < (unsigned long)high_memory;
1310	#endif
1311
1312	/*
1313	* Allocate the memory object.
1314	*/
1315	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, pv, cb);
1316	if (!pMemLnx)
1317	{
1318	IPRT_LINUX_RESTORE_EFL_AC();
1319	return VERR_NO_MEMORY;
1320	}
1321
1322	/*
1323	* Gather the pages.
1324	* We ASSUME all kernel pages are non-swappable and non-movable.
1325	*/
1326	rc = VINF_SUCCESS;
1327	pbPage = (uint8_t *)pvLast;
1328	iPage = cPages;
1329	if (!fLinearMapping)
1330	{
1331	while (iPage-- > 0)
1332	{
1333	struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1334	if (RT_UNLIKELY(!pPage))
1335	{
1336	rc = VERR_LOCK_FAILED;
1337	break;
1338	}
1339	pMemLnx->apPages[iPage] = pPage;
1340	pbPage -= PAGE_SIZE;
1341	}
1342	}
1343	else
1344	{
1345	while (iPage-- > 0)
1346	{
1347	pMemLnx->apPages[iPage] = virt_to_page(pbPage);
1348	pbPage -= PAGE_SIZE;
1349	}
1350	}
1351	if (RT_SUCCESS(rc))
1352	{
1353	/*
1354	* Complete the memory object and return.
1355	*/
1356	pMemLnx->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
1357	pMemLnx->cPages = cPages;
1358	Assert(!pMemLnx->fMappedToRing0);
1359	*ppMem = &pMemLnx->Core;
1360
1361	IPRT_LINUX_RESTORE_EFL_AC();
1362	return VINF_SUCCESS;
1363	}
1364
1365	rtR0MemObjDelete(&pMemLnx->Core);
1366	IPRT_LINUX_RESTORE_EFL_AC();
1367	return rc;
1368	}
1369
1370
1371	DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment)
1372	{
1373	#if RTLNX_VER_MIN(2,4,22)
1374	IPRT_LINUX_SAVE_EFL_AC();
1375	const size_t cPages = cb >> PAGE_SHIFT;
1376	struct page *pDummyPage;
1377	struct page **papPages;
1378
1379	/* check for unsupported stuff. */
1380	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1381	if (uAlignment > PAGE_SIZE)
1382	return VERR_NOT_SUPPORTED;
1383
1384	/*
1385	* Allocate a dummy page and create a page pointer array for vmap such that
1386	* the dummy page is mapped all over the reserved area.
1387	*/
1388	pDummyPage = alloc_page(GFP_HIGHUSER \| __GFP_NOWARN);
1389	if (pDummyPage)
1390	{
1391	papPages = RTMemAlloc(sizeof(papPages) cPages);
1392	if (papPages)
1393	{
1394	void *pv;
1395	size_t iPage = cPages;
1396	while (iPage-- > 0)
1397	papPages[iPage] = pDummyPage;
1398	# ifdef VM_MAP
1399	pv = vmap(papPages, cPages, VM_MAP, PAGE_KERNEL_RO);
1400	# else
1401	pv = vmap(papPages, cPages, VM_ALLOC, PAGE_KERNEL_RO);
1402	# endif
1403	RTMemFree(papPages);
1404	if (pv)
1405	{
1406	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1407	if (pMemLnx)
1408	{
1409	pMemLnx->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
1410	pMemLnx->cPages = 1;
1411	pMemLnx->apPages[0] = pDummyPage;
1412	*ppMem = &pMemLnx->Core;
1413	IPRT_LINUX_RESTORE_EFL_AC();
1414	return VINF_SUCCESS;
1415	}
1416	vunmap(pv);
1417	}
1418	}
1419	__free_page(pDummyPage);
1420	}
1421	IPRT_LINUX_RESTORE_EFL_AC();
1422	return VERR_NO_MEMORY;
1423
1424	#else /* < 2.4.22 */
1425	/*
1426	* Could probably use ioremap here, but the caller is in a better position than us
1427	* to select some safe physical memory.
1428	*/
1429	return VERR_NOT_SUPPORTED;
1430	#endif
1431	}
1432
1433
1434	DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process)
1435	{
1436	IPRT_LINUX_SAVE_EFL_AC();
1437	PRTR0MEMOBJLNX pMemLnx;
1438	void *pv;
1439	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1440	if (!pTask)
1441	return VERR_NOT_SUPPORTED;
1442
1443	/*
1444	* Check that the specified alignment is supported.
1445	*/
1446	if (uAlignment > PAGE_SIZE)
1447	return VERR_NOT_SUPPORTED;
1448
1449	/*
1450	* Let rtR0MemObjLinuxDoMmap do the difficult bits.
1451	*/
1452	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cb, uAlignment, pTask, RTMEM_PROT_NONE);
1453	if (pv == (void *)-1)
1454	{
1455	IPRT_LINUX_RESTORE_EFL_AC();
1456	return VERR_NO_MEMORY;
1457	}
1458
1459	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1460	if (!pMemLnx)
1461	{
1462	rtR0MemObjLinuxDoMunmap(pv, cb, pTask);
1463	IPRT_LINUX_RESTORE_EFL_AC();
1464	return VERR_NO_MEMORY;
1465	}
1466
1467	pMemLnx->Core.u.ResVirt.R0Process = R0Process;
1468	*ppMem = &pMemLnx->Core;
1469	IPRT_LINUX_RESTORE_EFL_AC();
1470	return VINF_SUCCESS;
1471	}
1472
1473
1474	DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap,
1475	void *pvFixed, size_t uAlignment,
1476	unsigned fProt, size_t offSub, size_t cbSub)
1477	{
1478	int rc = VERR_NO_MEMORY;
1479	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1480	PRTR0MEMOBJLNX pMemLnx;
1481	IPRT_LINUX_SAVE_EFL_AC();
1482
1483	/* Fail if requested to do something we can't. */
1484	AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1485	if (uAlignment > PAGE_SIZE)
1486	return VERR_NOT_SUPPORTED;
1487
1488	/*
1489	* Create the IPRT memory object.
1490	*/
1491	if (!cbSub)
1492	cbSub = pMemLnxToMap->Core.cb - offSub;
1493	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub);
1494	if (pMemLnx)
1495	{
1496	if (pMemLnxToMap->cPages)
1497	{
1498	#if RTLNX_VER_MIN(2,4,22)
1499	/*
1500	* Use vmap - 2.4.22 and later.
1501	*/
1502	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, true /* kernel */);
1503	/** @todo We don't really care too much for EXEC here... 5.8 always adds NX. */
1504	Assert(((offSub + cbSub) >> PAGE_SHIFT) <= pMemLnxToMap->cPages);
1505	# ifdef VM_MAP
1506	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_MAP, fPg);
1507	# else
1508	pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_ALLOC, fPg);
1509	# endif
1510	if (pMemLnx->Core.pv)
1511	{
1512	pMemLnx->fMappedToRing0 = true;
1513	rc = VINF_SUCCESS;
1514	}
1515	else
1516	rc = VERR_MAP_FAILED;
1517
1518	#else /* < 2.4.22 */
1519	/*
1520	* Only option here is to share mappings if possible and forget about fProt.
1521	*/
1522	if (rtR0MemObjIsRing3(pMemToMap))
1523	rc = VERR_NOT_SUPPORTED;
1524	else
1525	{
1526	rc = VINF_SUCCESS;
1527	if (!pMemLnxToMap->Core.pv)
1528	rc = rtR0MemObjLinuxVMap(pMemLnxToMap, !!(fProt & RTMEM_PROT_EXEC));
1529	if (RT_SUCCESS(rc))
1530	{
1531	Assert(pMemLnxToMap->Core.pv);
1532	pMemLnx->Core.pv = (uint8_t *)pMemLnxToMap->Core.pv + offSub;
1533	}
1534	}
1535	#endif
1536	}
1537	else
1538	{
1539	/*
1540	* MMIO / physical memory.
1541	*/
1542	Assert(pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS && !pMemLnxToMap->Core.u.Phys.fAllocated);
1543	#if RTLNX_VER_MIN(2,6,25)
1544	/*
1545	* ioremap() defaults to no caching since the 2.6 kernels.
1546	* ioremap_nocache() has been removed finally in 5.6-rc1.
1547	*/
1548	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1549	? ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1550	: ioremap_cache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1551	#else /* KERNEL_VERSION < 2.6.25 */
1552	pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1553	? ioremap_nocache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
1554	: ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
1555	#endif /* KERNEL_VERSION < 2.6.25 */
1556	if (pMemLnx->Core.pv)
1557	{
1558	/** @todo fix protection. */
1559	rc = VINF_SUCCESS;
1560	}
1561	}
1562	if (RT_SUCCESS(rc))
1563	{
1564	pMemLnx->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1565	*ppMem = &pMemLnx->Core;
1566	IPRT_LINUX_RESTORE_EFL_AC();
1567	return VINF_SUCCESS;
1568	}
1569	rtR0MemObjDelete(&pMemLnx->Core);
1570	}
1571
1572	IPRT_LINUX_RESTORE_EFL_AC();
1573	return rc;
1574	}
1575
1576
1577	#ifdef VBOX_USE_PAE_HACK
1578	/**
1579	* Replace the PFN of a PTE with the address of the actual page.
1580	*
1581	* The caller maps a reserved dummy page at the address with the desired access
1582	* and flags.
1583	*
1584	* This hack is required for older Linux kernels which don't provide
1585	* remap_pfn_range().
1586	*
1587	* @returns 0 on success, -ENOMEM on failure.
1588	* @param mm The memory context.
1589	* @param ulAddr The mapping address.
1590	* @param Phys The physical address of the page to map.
1591	*/
1592	static int rtR0MemObjLinuxFixPte(struct mm_struct *mm, unsigned long ulAddr, RTHCPHYS Phys)
1593	{
1594	int rc = -ENOMEM;
1595	pgd_t *pgd;
1596
1597	spin_lock(&mm->page_table_lock);
1598
1599	pgd = pgd_offset(mm, ulAddr);
1600	if (!pgd_none(pgd) && !pgd_bad(pgd))
1601	{
1602	pmd_t *pmd = pmd_offset(pgd, ulAddr);
1603	if (!pmd_none(*pmd))
1604	{
1605	pte_t *ptep = pte_offset_map(pmd, ulAddr);
1606	if (ptep)
1607	{
1608	pte_t pte = *ptep;
1609	pte.pte_high &= 0xfff00000;
1610	pte.pte_high \|= ((Phys >> 32) & 0x000fffff);
1611	pte.pte_low &= 0x00000fff;
1612	pte.pte_low \|= (Phys & 0xfffff000);
1613	set_pte(ptep, pte);
1614	pte_unmap(ptep);
1615	rc = 0;
1616	}
1617	}
1618	}
1619
1620	spin_unlock(&mm->page_table_lock);
1621	return rc;
1622	}
1623	#endif /* VBOX_USE_PAE_HACK */
1624
1625
1626	DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
1627	unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub)
1628	{
1629	struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1630	PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1631	int rc = VERR_NO_MEMORY;
1632	PRTR0MEMOBJLNX pMemLnx;
1633	#ifdef VBOX_USE_PAE_HACK
1634	struct page *pDummyPage;
1635	RTHCPHYS DummyPhys;
1636	#endif
1637	IPRT_LINUX_SAVE_EFL_AC();
1638
1639	/*
1640	* Check for restrictions.
1641	*/
1642	if (!pTask)
1643	return VERR_NOT_SUPPORTED;
1644	if (uAlignment > PAGE_SIZE)
1645	return VERR_NOT_SUPPORTED;
1646
1647	#ifdef VBOX_USE_PAE_HACK
1648	/*
1649	* Allocate a dummy page for use when mapping the memory.
1650	*/
1651	pDummyPage = alloc_page(GFP_USER \| __GFP_NOWARN);
1652	if (!pDummyPage)
1653	{
1654	IPRT_LINUX_RESTORE_EFL_AC();
1655	return VERR_NO_MEMORY;
1656	}
1657	SetPageReserved(pDummyPage);
1658	DummyPhys = page_to_phys(pDummyPage);
1659	#endif
1660
1661	/*
1662	* Create the IPRT memory object.
1663	*/
1664	Assert(!offSub \|\| cbSub);
1665	if (cbSub == 0)
1666	cbSub = pMemLnxToMap->Core.cb;
1667	pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub);
1668	if (pMemLnx)
1669	{
1670	/*
1671	* Allocate user space mapping.
1672	*/
1673	void *pv;
1674	pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cbSub, uAlignment, pTask, fProt);
1675	if (pv != (void *)-1)
1676	{
1677	/*
1678	* Map page by page into the mmap area.
1679	* This is generic, paranoid and not very efficient.
1680	*/
1681	pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, false /* user */);
1682	unsigned long ulAddrCur = (unsigned long)pv;
1683	const size_t cPages = (offSub + cbSub) >> PAGE_SHIFT;
1684	size_t iPage;
1685
1686	LNX_MM_DOWN_WRITE(pTask->mm);
1687
1688	rc = VINF_SUCCESS;
1689	if (pMemLnxToMap->cPages)
1690	{
1691	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE)
1692	{
1693	#if RTLNX_VER_MAX(2,6,11)
1694	RTHCPHYS Phys = page_to_phys(pMemLnxToMap->apPages[iPage]);
1695	#endif
1696	#if RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1697	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1698	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1699	#endif
1700	#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
1701	/* remap_page_range() limitation on x86 */
1702	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1703	#endif
1704
1705	#if defined(VBOX_USE_INSERT_PAGE) && RTLNX_VER_MIN(2,6,22)
1706	rc = vm_insert_page(vma, ulAddrCur, pMemLnxToMap->apPages[iPage]);
1707	/* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
1708	* See remap_pfn_range() in mm/memory.c */
1709	#if RTLNX_VER_MIN(3,7,0)
1710	vma->vm_flags \|= VM_DONTEXPAND \| VM_DONTDUMP;
1711	#else
1712	vma->vm_flags \|= VM_RESERVED;
1713	#endif
1714	#elif RTLNX_VER_MIN(2,6,11)
1715	rc = remap_pfn_range(vma, ulAddrCur, page_to_pfn(pMemLnxToMap->apPages[iPage]), PAGE_SIZE, fPg);
1716	#elif defined(VBOX_USE_PAE_HACK)
1717	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1718	if (!rc)
1719	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1720	#elif RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1721	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1722	#else /* 2.4 */
1723	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1724	#endif
1725	if (rc)
1726	{
1727	rc = VERR_NO_MEMORY;
1728	break;
1729	}
1730	}
1731	}
1732	else
1733	{
1734	RTHCPHYS Phys;
1735	if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS)
1736	Phys = pMemLnxToMap->Core.u.Phys.PhysBase;
1737	else if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_CONT)
1738	Phys = pMemLnxToMap->Core.u.Cont.Phys;
1739	else
1740	{
1741	AssertMsgFailed(("%d\n", pMemLnxToMap->Core.enmType));
1742	Phys = NIL_RTHCPHYS;
1743	}
1744	if (Phys != NIL_RTHCPHYS)
1745	{
1746	for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE, Phys += PAGE_SIZE)
1747	{
1748	#if RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1749	struct vm_area_struct vma = find_vma(pTask->mm, ulAddrCur); / this is probably the same for all the pages... */
1750	AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1751	#endif
1752	#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
1753	/* remap_page_range() limitation on x86 */
1754	AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1755	#endif
1756
1757	#if RTLNX_VER_MIN(2,6,11)
1758	rc = remap_pfn_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1759	#elif defined(VBOX_USE_PAE_HACK)
1760	rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1761	if (!rc)
1762	rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1763	#elif RTLNX_VER_MIN(2,6,0) \|\| defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1764	rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1765	#else /* 2.4 */
1766	rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1767	#endif
1768	if (rc)
1769	{
1770	rc = VERR_NO_MEMORY;
1771	break;
1772	}
1773	}
1774	}
1775	}
1776
1777	#ifdef CONFIG_NUMA_BALANCING
1778	# if RTLNX_VER_MAX(3,13,0)
1779	# ifdef RHEL_RELEASE_CODE
1780	# if RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7, 0)
1781	# define VBOX_NUMA_HACK_OLD
1782	# endif
1783	# endif
1784	# endif
1785	if (RT_SUCCESS(rc))
1786	{
1787	/** @todo Ugly hack! But right now we have no other means to
1788	* disable automatic NUMA page balancing. */
1789	# ifdef RT_OS_X86
1790	# ifdef VBOX_NUMA_HACK_OLD
1791	pTask->mm->numa_next_reset = jiffies + 0x7fffffffUL;
1792	# endif
1793	pTask->mm->numa_next_scan = jiffies + 0x7fffffffUL;
1794	# else
1795	# ifdef VBOX_NUMA_HACK_OLD
1796	pTask->mm->numa_next_reset = jiffies + 0x7fffffffffffffffUL;
1797	# endif
1798	pTask->mm->numa_next_scan = jiffies + 0x7fffffffffffffffUL;
1799	# endif
1800	}
1801	#endif /* CONFIG_NUMA_BALANCING */
1802
1803	LNX_MM_UP_WRITE(pTask->mm);
1804
1805	if (RT_SUCCESS(rc))
1806	{
1807	#ifdef VBOX_USE_PAE_HACK
1808	__free_page(pDummyPage);
1809	#endif
1810	pMemLnx->Core.pv = pv;
1811	pMemLnx->Core.u.Mapping.R0Process = R0Process;
1812	*ppMem = &pMemLnx->Core;
1813	IPRT_LINUX_RESTORE_EFL_AC();
1814	return VINF_SUCCESS;
1815	}
1816
1817	/*
1818	* Bail out.
1819	*/
1820	rtR0MemObjLinuxDoMunmap(pv, cbSub, pTask);
1821	}
1822	rtR0MemObjDelete(&pMemLnx->Core);
1823	}
1824	#ifdef VBOX_USE_PAE_HACK
1825	__free_page(pDummyPage);
1826	#endif
1827
1828	IPRT_LINUX_RESTORE_EFL_AC();
1829	return rc;
1830	}
1831
1832
1833	DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1834	{
1835	# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
1836	/*
1837	* Currently only supported when we've got addresses PTEs from the kernel.
1838	*/
1839	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1840	if (pMemLnx->pArea && pMemLnx->papPtesForArea)
1841	{
1842	pgprot_t const fPg = rtR0MemObjLinuxConvertProt(fProt, true /fKernel/);
1843	size_t const cPages = (offSub + cbSub) >> PAGE_SHIFT;
1844	pte_t **papPtes = pMemLnx->papPtesForArea;
1845	size_t i;
1846
1847	for (i = offSub >> PAGE_SHIFT; i < cPages; i++)
1848	{
1849	set_pte(papPtes[i], mk_pte(pMemLnx->apPages[i], fPg));
1850	}
1851	preempt_disable();
1852	__flush_tlb_all();
1853	preempt_enable();
1854	return VINF_SUCCESS;
1855	}
1856	# endif
1857
1858	NOREF(pMem);
1859	NOREF(offSub);
1860	NOREF(cbSub);
1861	NOREF(fProt);
1862	return VERR_NOT_SUPPORTED;
1863	}
1864
1865
1866	DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1867	{
1868	PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1869
1870	if (pMemLnx->cPages)
1871	return page_to_phys(pMemLnx->apPages[iPage]);
1872
1873	switch (pMemLnx->Core.enmType)
1874	{
1875	case RTR0MEMOBJTYPE_CONT:
1876	return pMemLnx->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);
1877
1878	case RTR0MEMOBJTYPE_PHYS:
1879	return pMemLnx->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);
1880
1881	/* the parent knows */
1882	case RTR0MEMOBJTYPE_MAPPING:
1883	return rtR0MemObjNativeGetPagePhysAddr(pMemLnx->Core.uRel.Child.pParent, iPage);
1884
1885	/* cPages > 0 */
1886	case RTR0MEMOBJTYPE_LOW:
1887	case RTR0MEMOBJTYPE_LOCK:
1888	case RTR0MEMOBJTYPE_PHYS_NC:
1889	case RTR0MEMOBJTYPE_PAGE:
1890	default:
1891	AssertMsgFailed(("%d\n", pMemLnx->Core.enmType));
1892	/* fall thru */
1893
1894	case RTR0MEMOBJTYPE_RES_VIRT:
1895	return NIL_RTHCPHYS;
1896	}
1897	}
1898

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c@ 85698

Download in other formats: