1 | /* $Id: GMMR0.cpp 106061 2024-09-16 14:03:52Z vboxsync $ */
|
---|
2 | /** @file
|
---|
3 | * GMM - Global Memory Manager.
|
---|
4 | */
|
---|
5 |
|
---|
6 | /*
|
---|
7 | * Copyright (C) 2007-2024 Oracle and/or its affiliates.
|
---|
8 | *
|
---|
9 | * This file is part of VirtualBox base platform packages, as
|
---|
10 | * available from https://www.virtualbox.org.
|
---|
11 | *
|
---|
12 | * This program is free software; you can redistribute it and/or
|
---|
13 | * modify it under the terms of the GNU General Public License
|
---|
14 | * as published by the Free Software Foundation, in version 3 of the
|
---|
15 | * License.
|
---|
16 | *
|
---|
17 | * This program is distributed in the hope that it will be useful, but
|
---|
18 | * WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
19 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
---|
20 | * General Public License for more details.
|
---|
21 | *
|
---|
22 | * You should have received a copy of the GNU General Public License
|
---|
23 | * along with this program; if not, see <https://www.gnu.org/licenses>.
|
---|
24 | *
|
---|
25 | * SPDX-License-Identifier: GPL-3.0-only
|
---|
26 | */
|
---|
27 |
|
---|
28 |
|
---|
29 | /** @page pg_gmm GMM - The Global Memory Manager
|
---|
30 | *
|
---|
31 | * As the name indicates, this component is responsible for global memory
|
---|
32 | * management. Currently only guest RAM is allocated from the GMM, but this
|
---|
33 | * may change to include shadow page tables and other bits later.
|
---|
34 | *
|
---|
35 | * Guest RAM is managed as individual pages, but allocated from the host OS
|
---|
36 | * in chunks for reasons of portability / efficiency. To minimize the memory
|
---|
37 | * footprint all tracking structure must be as small as possible without
|
---|
38 | * unnecessary performance penalties.
|
---|
39 | *
|
---|
40 | * The allocation chunks has fixed sized, the size defined at compile time
|
---|
41 | * by the #GMM_CHUNK_SIZE \#define.
|
---|
42 | *
|
---|
43 | * Each chunk is given an unique ID. Each page also has a unique ID. The
|
---|
44 | * relationship between the two IDs is:
|
---|
45 | * @code
|
---|
46 | * GMM_CHUNK_SHIFT = log2(GMM_CHUNK_SIZE / GUEST_PAGE_SIZE);
|
---|
47 | * idPage = (idChunk << GMM_CHUNK_SHIFT) | iPage;
|
---|
48 | * @endcode
|
---|
49 | * Where iPage is the index of the page within the chunk. This ID scheme
|
---|
50 | * permits for efficient chunk and page lookup, but it relies on the chunk size
|
---|
51 | * to be set at compile time. The chunks are organized in an AVL tree with their
|
---|
52 | * IDs being the keys.
|
---|
53 | *
|
---|
54 | * @todo Scope the chunk+page IDs based on config setting: per VM,
|
---|
55 | * per user (default), or global. This will prevent ring-3 code screwing
|
---|
56 | * around with random page IDs from accessing someone else's data in the
|
---|
57 | * default config. This would let us move HCPhys out of PGMPAGE when
|
---|
58 | * restricting it to ring-0 only, w/o requiring any additional ring-0 per
|
---|
59 | * page data (prereq mmio2 must go via GMM). See @bugref{10696} for more.
|
---|
60 | *
|
---|
61 | * The physical address of each page in an allocation chunk is maintained by
|
---|
62 | * the #RTR0MEMOBJ and obtained using #RTR0MemObjGetPagePhysAddr. There is no
|
---|
63 | * need to duplicate this information (it'll cost 8-bytes per page if we did).
|
---|
64 | *
|
---|
65 | * So what do we need to track per page? Most importantly we need to know
|
---|
66 | * which state the page is in:
|
---|
67 | * - Private - Allocated for (eventually) backing one particular VM page.
|
---|
68 | * - Shared - Readonly page that is used by one or more VMs and treated
|
---|
69 | * as COW by PGM.
|
---|
70 | * - Free - Not used by anyone.
|
---|
71 | *
|
---|
72 | * For the page replacement operations (sharing, defragmenting and freeing)
|
---|
73 | * to be somewhat efficient, private pages needs to be associated with a
|
---|
74 | * particular page in a particular VM.
|
---|
75 | *
|
---|
76 | * Tracking the usage of shared pages is impractical and expensive, so we'll
|
---|
77 | * settle for a reference counting system instead.
|
---|
78 | *
|
---|
79 | * Free pages will be chained on LIFOs
|
---|
80 | *
|
---|
81 | * On 64-bit systems we will use a 64-bit bitfield per page, while on 32-bit
|
---|
82 | * systems a 32-bit bitfield will have to suffice because of address space
|
---|
83 | * limitations. The #GMMPAGE structure shows the details.
|
---|
84 | *
|
---|
85 | *
|
---|
86 | * @section sec_gmm_alloc_strat Page Allocation Strategy
|
---|
87 | *
|
---|
88 | * The strategy for allocating pages has to take fragmentation and shared
|
---|
89 | * pages into account, or we may end up with with 2000 chunks with only
|
---|
90 | * a few pages in each. Shared pages cannot easily be reallocated because
|
---|
91 | * of the inaccurate usage accounting (see above). Private pages can be
|
---|
92 | * reallocated by a defragmentation thread in the same manner that sharing
|
---|
93 | * is done.
|
---|
94 | *
|
---|
95 | * The first approach is to manage the free pages in two sets depending on
|
---|
96 | * whether they are mainly for the allocation of shared or private pages.
|
---|
97 | * In the initial implementation there will be almost no possibility for
|
---|
98 | * mixing shared and private pages in the same chunk (only if we're really
|
---|
99 | * stressed on memory), but when we implement forking of VMs and have to
|
---|
100 | * deal with lots of COW pages it'll start getting kind of interesting.
|
---|
101 | *
|
---|
102 | * The sets are lists of chunks with approximately the same number of
|
---|
103 | * free pages. Say the chunk size is 1MB, meaning 256 pages, and a set
|
---|
104 | * consists of 16 lists. So, the first list will contain the chunks with
|
---|
105 | * 1-7 free pages, the second covers 8-15, and so on. The chunks will be
|
---|
106 | * moved between the lists as pages are freed up or allocated.
|
---|
107 | *
|
---|
108 | *
|
---|
109 | * @section sec_gmm_costs Costs
|
---|
110 | *
|
---|
111 | * The per page cost in kernel space is 32-bit plus whatever RTR0MEMOBJ
|
---|
112 | * entails. In addition there is the chunk cost of approximately
|
---|
113 | * (sizeof(RT0MEMOBJ) + sizeof(CHUNK)) / 2^CHUNK_SHIFT bytes per page.
|
---|
114 | *
|
---|
115 | * On Windows the per page #RTR0MEMOBJ cost is 32-bit on 32-bit windows
|
---|
116 | * and 64-bit on 64-bit windows (a PFN_NUMBER in the MDL). So, 64-bit per page.
|
---|
117 | * The cost on Linux is identical, but here it's because of sizeof(struct page *).
|
---|
118 | *
|
---|
119 | *
|
---|
120 | * @section sec_gmm_legacy Legacy Mode for Non-Tier-1 Platforms
|
---|
121 | *
|
---|
122 | * In legacy mode the page source is locked user pages and not
|
---|
123 | * #RTR0MemObjAllocPhysNC, this means that a page can only be allocated
|
---|
124 | * by the VM that locked it. We will make no attempt at implementing
|
---|
125 | * page sharing on these systems, just do enough to make it all work.
|
---|
126 | *
|
---|
127 | * @note With 6.1 really dropping 32-bit support, the legacy mode is obsoleted
|
---|
128 | * under the assumption that there is sufficient kernel virtual address
|
---|
129 | * space to map all of the guest memory allocations. So, we'll be using
|
---|
130 | * #RTR0MemObjAllocPage on some platforms as an alternative to
|
---|
131 | * #RTR0MemObjAllocPhysNC.
|
---|
132 | *
|
---|
133 | *
|
---|
134 | * @subsection sub_gmm_locking Serializing
|
---|
135 | *
|
---|
136 | * One simple fast mutex will be employed in the initial implementation, not
|
---|
137 | * two as mentioned in @ref sec_pgmPhys_Serializing.
|
---|
138 | *
|
---|
139 | * @see @ref sec_pgmPhys_Serializing
|
---|
140 | *
|
---|
141 | *
|
---|
142 | * @section sec_gmm_overcommit Memory Over-Commitment Management
|
---|
143 | *
|
---|
144 | * The GVM will have to do the system wide memory over-commitment
|
---|
145 | * management. My current ideas are:
|
---|
146 | * - Per VM oc policy that indicates how much to initially commit
|
---|
147 | * to it and what to do in a out-of-memory situation.
|
---|
148 | * - Prevent overtaxing the host.
|
---|
149 | *
|
---|
150 | * There are some challenges here, the main ones are configurability and
|
---|
151 | * security. Should we for instance permit anyone to request 100% memory
|
---|
152 | * commitment? Who should be allowed to do runtime adjustments of the
|
---|
153 | * config. And how to prevent these settings from being lost when the last
|
---|
154 | * VM process exits? The solution is probably to have an optional root
|
---|
155 | * daemon the will keep VMMR0.r0 in memory and enable the security measures.
|
---|
156 | *
|
---|
157 | *
|
---|
158 | *
|
---|
159 | * @section sec_gmm_numa NUMA
|
---|
160 | *
|
---|
161 | * NUMA considerations will be designed and implemented a bit later.
|
---|
162 | *
|
---|
163 | * The preliminary guesses is that we will have to try allocate memory as
|
---|
164 | * close as possible to the CPUs the VM is executed on (EMT and additional CPU
|
---|
165 | * threads). Which means it's mostly about allocation and sharing policies.
|
---|
166 | * Both the scheduler and allocator interface will to supply some NUMA info
|
---|
167 | * and we'll need to have a way to calc access costs.
|
---|
168 | *
|
---|
169 | */
|
---|
170 |
|
---|
171 |
|
---|
172 | /*********************************************************************************************************************************
|
---|
173 | * Header Files *
|
---|
174 | *********************************************************************************************************************************/
|
---|
175 | #define LOG_GROUP LOG_GROUP_GMM
|
---|
176 | #include <VBox/rawpci.h>
|
---|
177 | #include <VBox/vmm/gmm.h>
|
---|
178 | #include "GMMR0Internal.h"
|
---|
179 | #include <VBox/vmm/vmcc.h>
|
---|
180 | #include <VBox/vmm/pgm.h>
|
---|
181 | #include <VBox/log.h>
|
---|
182 | #include <VBox/param.h>
|
---|
183 | #include <VBox/err.h>
|
---|
184 | #include <VBox/VMMDev.h>
|
---|
185 | #include <iprt/asm.h>
|
---|
186 | #include <iprt/avl.h>
|
---|
187 | #ifdef VBOX_STRICT
|
---|
188 | # include <iprt/crc.h>
|
---|
189 | #endif
|
---|
190 | #include <iprt/critsect.h>
|
---|
191 | #include <iprt/list.h>
|
---|
192 | #include <iprt/mem.h>
|
---|
193 | #include <iprt/memobj.h>
|
---|
194 | #include <iprt/mp.h>
|
---|
195 | #include <iprt/semaphore.h>
|
---|
196 | #include <iprt/spinlock.h>
|
---|
197 | #include <iprt/string.h>
|
---|
198 | #include <iprt/time.h>
|
---|
199 |
|
---|
200 | /* This is 64-bit only code now. */
|
---|
201 | #if HC_ARCH_BITS != 64 || ARCH_BITS != 64
|
---|
202 | # error "This is 64-bit only code"
|
---|
203 | #endif
|
---|
204 |
|
---|
205 |
|
---|
206 | /*********************************************************************************************************************************
|
---|
207 | * Defined Constants And Macros *
|
---|
208 | *********************************************************************************************************************************/
|
---|
209 | /** @def VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
210 | * Use a critical section instead of a fast mutex for the giant GMM lock.
|
---|
211 | *
|
---|
212 | * @remarks This is primarily a way of avoiding the deadlock checks in the
|
---|
213 | * windows driver verifier. */
|
---|
214 | #if defined(RT_OS_WINDOWS) || defined(RT_OS_DARWIN) || defined(DOXYGEN_RUNNING)
|
---|
215 | # define VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
216 | #endif
|
---|
217 |
|
---|
218 |
|
---|
219 | /*********************************************************************************************************************************
|
---|
220 | * Structures and Typedefs *
|
---|
221 | *********************************************************************************************************************************/
|
---|
222 | /** Pointer to set of free chunks. */
|
---|
223 | typedef struct GMMCHUNKFREESET *PGMMCHUNKFREESET;
|
---|
224 |
|
---|
225 | /**
|
---|
226 | * The per-page tracking structure employed by the GMM.
|
---|
227 | *
|
---|
228 | * Because of the different layout on 32-bit and 64-bit hosts in earlier
|
---|
229 | * versions of the code, macros are used to get and set some of the data.
|
---|
230 | */
|
---|
231 | typedef union GMMPAGE
|
---|
232 | {
|
---|
233 | /** Unsigned integer view. */
|
---|
234 | uint64_t u;
|
---|
235 |
|
---|
236 | /** The common view. */
|
---|
237 | struct GMMPAGECOMMON
|
---|
238 | {
|
---|
239 | uint32_t uStuff1 : 32;
|
---|
240 | uint32_t uStuff2 : 30;
|
---|
241 | /** The page state. */
|
---|
242 | uint32_t u2State : 2;
|
---|
243 | } Common;
|
---|
244 |
|
---|
245 | /** The view of a private page. */
|
---|
246 | struct GMMPAGEPRIVATE
|
---|
247 | {
|
---|
248 | /** The guest page frame number. (Max addressable: 2 ^ 44 - 16) */
|
---|
249 | uint32_t pfn;
|
---|
250 | /** The GVM handle. (64K VMs) */
|
---|
251 | uint32_t hGVM : 16;
|
---|
252 | /** Reserved. */
|
---|
253 | uint32_t u16Reserved : 14;
|
---|
254 | /** The page state. */
|
---|
255 | uint32_t u2State : 2;
|
---|
256 | } Private;
|
---|
257 |
|
---|
258 | /** The view of a shared page. */
|
---|
259 | struct GMMPAGESHARED
|
---|
260 | {
|
---|
261 | /** The host page frame number. (Max addressable: 2 ^ 44 - 16) */
|
---|
262 | uint32_t pfn;
|
---|
263 | /** The reference count (64K VMs). */
|
---|
264 | uint32_t cRefs : 16;
|
---|
265 | /** Used for debug checksumming. */
|
---|
266 | uint32_t u14Checksum : 14;
|
---|
267 | /** The page state. */
|
---|
268 | uint32_t u2State : 2;
|
---|
269 | } Shared;
|
---|
270 |
|
---|
271 | /** The view of a free page. */
|
---|
272 | struct GMMPAGEFREE
|
---|
273 | {
|
---|
274 | /** The index of the next page in the free list. UINT16_MAX is NIL. */
|
---|
275 | uint16_t iNext;
|
---|
276 | /** Reserved. Checksum or something? */
|
---|
277 | uint16_t u16Reserved0;
|
---|
278 | /** Reserved. Checksum or something? */
|
---|
279 | uint32_t u30Reserved1 : 29;
|
---|
280 | /** Set if the page was zeroed. */
|
---|
281 | uint32_t fZeroed : 1;
|
---|
282 | /** The page state. */
|
---|
283 | uint32_t u2State : 2;
|
---|
284 | } Free;
|
---|
285 | } GMMPAGE;
|
---|
286 | AssertCompileSize(GMMPAGE, sizeof(RTHCUINTPTR));
|
---|
287 | /** Pointer to a GMMPAGE. */
|
---|
288 | typedef GMMPAGE *PGMMPAGE;
|
---|
289 |
|
---|
290 |
|
---|
291 | /** @name The Page States.
|
---|
292 | * @{ */
|
---|
293 | /** A private page. */
|
---|
294 | #define GMM_PAGE_STATE_PRIVATE 0
|
---|
295 | /** A shared page. */
|
---|
296 | #define GMM_PAGE_STATE_SHARED 2
|
---|
297 | /** A free page. */
|
---|
298 | #define GMM_PAGE_STATE_FREE 3
|
---|
299 | /** @} */
|
---|
300 |
|
---|
301 |
|
---|
302 | /** @def GMM_PAGE_IS_PRIVATE
|
---|
303 | *
|
---|
304 | * @returns true if private, false if not.
|
---|
305 | * @param pPage The GMM page.
|
---|
306 | */
|
---|
307 | #define GMM_PAGE_IS_PRIVATE(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_PRIVATE )
|
---|
308 |
|
---|
309 | /** @def GMM_PAGE_IS_SHARED
|
---|
310 | *
|
---|
311 | * @returns true if shared, false if not.
|
---|
312 | * @param pPage The GMM page.
|
---|
313 | */
|
---|
314 | #define GMM_PAGE_IS_SHARED(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_SHARED )
|
---|
315 |
|
---|
316 | /** @def GMM_PAGE_IS_FREE
|
---|
317 | *
|
---|
318 | * @returns true if free, false if not.
|
---|
319 | * @param pPage The GMM page.
|
---|
320 | */
|
---|
321 | #define GMM_PAGE_IS_FREE(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_FREE )
|
---|
322 |
|
---|
323 | /** @def GMM_PAGE_PFN_LAST
|
---|
324 | * The last valid guest pfn range.
|
---|
325 | * @remark Some of the values outside the range has special meaning,
|
---|
326 | * see GMM_PAGE_PFN_UNSHAREABLE.
|
---|
327 | */
|
---|
328 | #define GMM_PAGE_PFN_LAST UINT32_C(0xfffffff0)
|
---|
329 | AssertCompile(GMM_PAGE_PFN_LAST == (GMM_GCPHYS_LAST >> GUEST_PAGE_SHIFT));
|
---|
330 |
|
---|
331 | /** @def GMM_PAGE_PFN_UNSHAREABLE
|
---|
332 | * Indicates that this page isn't used for normal guest memory and thus isn't shareable.
|
---|
333 | */
|
---|
334 | #define GMM_PAGE_PFN_UNSHAREABLE UINT32_C(0xfffffff1)
|
---|
335 | AssertCompile(GMM_PAGE_PFN_UNSHAREABLE == (GMM_GCPHYS_UNSHAREABLE >> GUEST_PAGE_SHIFT));
|
---|
336 |
|
---|
337 |
|
---|
338 | /**
|
---|
339 | * A GMM allocation chunk ring-3 mapping record.
|
---|
340 | *
|
---|
341 | * This should really be associated with a session and not a VM, but
|
---|
342 | * it's simpler to associated with a VM and cleanup with the VM object
|
---|
343 | * is destroyed.
|
---|
344 | */
|
---|
345 | typedef struct GMMCHUNKMAP
|
---|
346 | {
|
---|
347 | /** The mapping object. */
|
---|
348 | RTR0MEMOBJ hMapObj;
|
---|
349 | /** The VM owning the mapping. */
|
---|
350 | PGVM pGVM;
|
---|
351 | } GMMCHUNKMAP;
|
---|
352 | /** Pointer to a GMM allocation chunk mapping. */
|
---|
353 | typedef struct GMMCHUNKMAP *PGMMCHUNKMAP;
|
---|
354 |
|
---|
355 |
|
---|
356 | /**
|
---|
357 | * A GMM allocation chunk.
|
---|
358 | */
|
---|
359 | typedef struct GMMCHUNK
|
---|
360 | {
|
---|
361 | /** The AVL node core.
|
---|
362 | * The Key is the chunk ID. (Giant mtx.) */
|
---|
363 | AVLU32NODECORE Core;
|
---|
364 | /** The memory object.
|
---|
365 | * Either from RTR0MemObjAllocPhysNC or RTR0MemObjLockUser depending on
|
---|
366 | * what the host can dish up with. (Chunk mtx protects mapping accesses
|
---|
367 | * and related frees.) */
|
---|
368 | RTR0MEMOBJ hMemObj;
|
---|
369 | #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
370 | /** Pointer to the kernel mapping. */
|
---|
371 | uint8_t *pbMapping;
|
---|
372 | #endif
|
---|
373 | /** Pointer to the next chunk in the free list. (Giant mtx.) */
|
---|
374 | PGMMCHUNK pFreeNext;
|
---|
375 | /** Pointer to the previous chunk in the free list. (Giant mtx.) */
|
---|
376 | PGMMCHUNK pFreePrev;
|
---|
377 | /** Pointer to the free set this chunk belongs to. NULL for
|
---|
378 | * chunks with no free pages. (Giant mtx.) */
|
---|
379 | PGMMCHUNKFREESET pSet;
|
---|
380 | /** List node in the chunk list (GMM::ChunkList). (Giant mtx.) */
|
---|
381 | RTLISTNODE ListNode;
|
---|
382 | /** Pointer to an array of mappings. (Chunk mtx.) */
|
---|
383 | PGMMCHUNKMAP paMappingsX;
|
---|
384 | /** The number of mappings. (Chunk mtx.) */
|
---|
385 | uint16_t cMappingsX;
|
---|
386 | /** The mapping lock this chunk is using using. UINT8_MAX if nobody is mapping
|
---|
387 | * or freeing anything. (Giant mtx.) */
|
---|
388 | uint8_t volatile iChunkMtx;
|
---|
389 | /** GMM_CHUNK_FLAGS_XXX. (Giant mtx.) */
|
---|
390 | uint8_t fFlags;
|
---|
391 | /** The head of the list of free pages. UINT16_MAX is the NIL value.
|
---|
392 | * (Giant mtx.) */
|
---|
393 | uint16_t iFreeHead;
|
---|
394 | /** The number of free pages. (Giant mtx.) */
|
---|
395 | uint16_t cFree;
|
---|
396 | /** The GVM handle of the VM that first allocated pages from this chunk, this
|
---|
397 | * is used as a preference when there are several chunks to choose from.
|
---|
398 | * When in bound memory mode this isn't a preference any longer. (Giant
|
---|
399 | * mtx.) */
|
---|
400 | uint16_t hGVM;
|
---|
401 | /** The ID of the NUMA node the memory mostly resides on. (Reserved for
|
---|
402 | * future use.) (Giant mtx.) */
|
---|
403 | uint16_t idNumaNode;
|
---|
404 | /** The number of private pages. (Giant mtx.) */
|
---|
405 | uint16_t cPrivate;
|
---|
406 | /** The number of shared pages. (Giant mtx.) */
|
---|
407 | uint16_t cShared;
|
---|
408 | /** The UID this chunk is associated with. */
|
---|
409 | RTUID uidOwner;
|
---|
410 | uint32_t u32Padding;
|
---|
411 | /** The pages. (Giant mtx.) */
|
---|
412 | GMMPAGE aPages[GMM_CHUNK_NUM_PAGES];
|
---|
413 | } GMMCHUNK;
|
---|
414 |
|
---|
415 | /** Indicates that the NUMA properies of the memory is unknown. */
|
---|
416 | #define GMM_CHUNK_NUMA_ID_UNKNOWN UINT16_C(0xfffe)
|
---|
417 |
|
---|
418 | /** @name GMM_CHUNK_FLAGS_XXX - chunk flags.
|
---|
419 | * @{ */
|
---|
420 | /** Indicates that the chunk is a large page (2MB). */
|
---|
421 | #define GMM_CHUNK_FLAGS_LARGE_PAGE UINT16_C(0x0001)
|
---|
422 | /** @} */
|
---|
423 |
|
---|
424 |
|
---|
425 | /**
|
---|
426 | * An allocation chunk TLB entry.
|
---|
427 | */
|
---|
428 | typedef struct GMMCHUNKTLBE
|
---|
429 | {
|
---|
430 | /** The chunk id. */
|
---|
431 | uint32_t idChunk;
|
---|
432 | /** Pointer to the chunk. */
|
---|
433 | PGMMCHUNK pChunk;
|
---|
434 | } GMMCHUNKTLBE;
|
---|
435 | /** Pointer to an allocation chunk TLB entry. */
|
---|
436 | typedef GMMCHUNKTLBE *PGMMCHUNKTLBE;
|
---|
437 |
|
---|
438 |
|
---|
439 | /** The number of entries in the allocation chunk TLB. */
|
---|
440 | #define GMM_CHUNKTLB_ENTRIES 32
|
---|
441 | /** Gets the TLB entry index for the given Chunk ID. */
|
---|
442 | #define GMM_CHUNKTLB_IDX(idChunk) ( (idChunk) & (GMM_CHUNKTLB_ENTRIES - 1) )
|
---|
443 |
|
---|
444 | /**
|
---|
445 | * An allocation chunk TLB.
|
---|
446 | */
|
---|
447 | typedef struct GMMCHUNKTLB
|
---|
448 | {
|
---|
449 | /** The TLB entries. */
|
---|
450 | GMMCHUNKTLBE aEntries[GMM_CHUNKTLB_ENTRIES];
|
---|
451 | } GMMCHUNKTLB;
|
---|
452 | /** Pointer to an allocation chunk TLB. */
|
---|
453 | typedef GMMCHUNKTLB *PGMMCHUNKTLB;
|
---|
454 |
|
---|
455 |
|
---|
456 | /**
|
---|
457 | * The GMM instance data.
|
---|
458 | */
|
---|
459 | typedef struct GMM
|
---|
460 | {
|
---|
461 | /** Magic / eye catcher. GMM_MAGIC */
|
---|
462 | uint32_t u32Magic;
|
---|
463 | /** The number of threads waiting on the mutex. */
|
---|
464 | uint32_t cMtxContenders;
|
---|
465 | #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
466 | /** The critical section protecting the GMM.
|
---|
467 | * More fine grained locking can be implemented later if necessary. */
|
---|
468 | RTCRITSECT GiantCritSect;
|
---|
469 | #else
|
---|
470 | /** The fast mutex protecting the GMM.
|
---|
471 | * More fine grained locking can be implemented later if necessary. */
|
---|
472 | RTSEMFASTMUTEX hMtx;
|
---|
473 | #endif
|
---|
474 | #ifdef VBOX_STRICT
|
---|
475 | /** The current mutex owner. */
|
---|
476 | RTNATIVETHREAD hMtxOwner;
|
---|
477 | #endif
|
---|
478 | /** Spinlock protecting the AVL tree.
|
---|
479 | * @todo Make this a read-write spinlock as we should allow concurrent
|
---|
480 | * lookups. */
|
---|
481 | RTSPINLOCK hSpinLockTree;
|
---|
482 | /** The chunk tree.
|
---|
483 | * Protected by hSpinLockTree. */
|
---|
484 | PAVLU32NODECORE pChunks;
|
---|
485 | /** Chunk freeing generation - incremented whenever a chunk is freed. Used
|
---|
486 | * for validating the per-VM chunk TLB entries. Valid range is 1 to 2^62
|
---|
487 | * (exclusive), though higher numbers may temporarily occure while
|
---|
488 | * invalidating the individual TLBs during wrap-around processing. */
|
---|
489 | uint64_t volatile idFreeGeneration;
|
---|
490 | /** The chunk TLB.
|
---|
491 | * Protected by hSpinLockTree. */
|
---|
492 | GMMCHUNKTLB ChunkTLB;
|
---|
493 | /** The private free set. */
|
---|
494 | GMMCHUNKFREESET PrivateX;
|
---|
495 | /** The shared free set. */
|
---|
496 | GMMCHUNKFREESET Shared;
|
---|
497 |
|
---|
498 | /** Shared module tree (global).
|
---|
499 | * @todo separate trees for distinctly different guest OSes. */
|
---|
500 | PAVLLU32NODECORE pGlobalSharedModuleTree;
|
---|
501 | /** Sharable modules (count of nodes in pGlobalSharedModuleTree). */
|
---|
502 | uint32_t cShareableModules;
|
---|
503 |
|
---|
504 | /** The chunk list. For simplifying the cleanup process and avoid tree
|
---|
505 | * traversal. */
|
---|
506 | RTLISTANCHOR ChunkList;
|
---|
507 |
|
---|
508 | /** The maximum number of pages we're allowed to allocate.
|
---|
509 | * @gcfgm{GMM/MaxPages,64-bit, Direct.}
|
---|
510 | * @gcfgm{GMM/PctPages,32-bit, Relative to the number of host pages.} */
|
---|
511 | uint64_t cMaxPages;
|
---|
512 | /** The number of pages that has been reserved.
|
---|
513 | * The deal is that cReservedPages - cOverCommittedPages <= cMaxPages. */
|
---|
514 | uint64_t cReservedPages;
|
---|
515 | /** The number of pages that we have over-committed in reservations. */
|
---|
516 | uint64_t cOverCommittedPages;
|
---|
517 | /** The number of actually allocated (committed if you like) pages. */
|
---|
518 | uint64_t cAllocatedPages;
|
---|
519 | /** The number of pages that are shared. A subset of cAllocatedPages. */
|
---|
520 | uint64_t cSharedPages;
|
---|
521 | /** The number of pages that are actually shared between VMs. */
|
---|
522 | uint64_t cDuplicatePages;
|
---|
523 | /** The number of pages that are shared that has been left behind by
|
---|
524 | * VMs not doing proper cleanups. */
|
---|
525 | uint64_t cLeftBehindSharedPages;
|
---|
526 | /** The number of allocation chunks.
|
---|
527 | * (The number of pages we've allocated from the host can be derived from this.) */
|
---|
528 | uint32_t cChunks;
|
---|
529 | /** The number of current ballooned pages. */
|
---|
530 | uint64_t cBalloonedPages;
|
---|
531 |
|
---|
532 | #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
533 | /** Whether #RTR0MemObjAllocPhysNC works. */
|
---|
534 | bool fHasWorkingAllocPhysNC;
|
---|
535 | #else
|
---|
536 | bool fPadding;
|
---|
537 | #endif
|
---|
538 | /** The bound memory mode indicator.
|
---|
539 | * When set, the memory will be bound to a specific VM and never
|
---|
540 | * shared. This is always set if fLegacyAllocationMode is set.
|
---|
541 | * (Also determined at initialization time.) */
|
---|
542 | bool fBoundMemoryMode;
|
---|
543 | /** The number of registered VMs. */
|
---|
544 | uint16_t cRegisteredVMs;
|
---|
545 |
|
---|
546 | /** The index of the next mutex to use. */
|
---|
547 | uint32_t iNextChunkMtx;
|
---|
548 | /** Chunk locks for reducing lock contention without having to allocate
|
---|
549 | * one lock per chunk. */
|
---|
550 | struct
|
---|
551 | {
|
---|
552 | /** The mutex */
|
---|
553 | RTSEMFASTMUTEX hMtx;
|
---|
554 | /** The number of threads currently using this mutex. */
|
---|
555 | uint32_t volatile cUsers;
|
---|
556 | } aChunkMtx[64];
|
---|
557 |
|
---|
558 | /** The number of freed chunks ever. This is used as list generation to
|
---|
559 | * avoid restarting the cleanup scanning when the list wasn't modified. */
|
---|
560 | uint32_t volatile cFreedChunks;
|
---|
561 | /** The previous allocated Chunk ID.
|
---|
562 | * Used as a hint to avoid scanning the whole bitmap. */
|
---|
563 | uint32_t idChunkPrev;
|
---|
564 | /** Spinlock protecting idChunkPrev & bmChunkId. */
|
---|
565 | RTSPINLOCK hSpinLockChunkId;
|
---|
566 | /** Chunk ID allocation bitmap.
|
---|
567 | * Bits of allocated IDs are set, free ones are clear.
|
---|
568 | * The NIL id (0) is marked allocated. */
|
---|
569 | uint32_t bmChunkId[(GMM_CHUNKID_LAST + 1 + 31) / 32];
|
---|
570 | } GMM;
|
---|
571 | /** Pointer to the GMM instance. */
|
---|
572 | typedef GMM *PGMM;
|
---|
573 |
|
---|
574 | /** The value of GMM::u32Magic (Katsuhiro Otomo). */
|
---|
575 | #define GMM_MAGIC UINT32_C(0x19540414)
|
---|
576 |
|
---|
577 |
|
---|
578 | /**
|
---|
579 | * GMM chunk mutex state.
|
---|
580 | *
|
---|
581 | * This is returned by gmmR0ChunkMutexAcquire and is used by the other
|
---|
582 | * gmmR0ChunkMutex* methods.
|
---|
583 | */
|
---|
584 | typedef struct GMMR0CHUNKMTXSTATE
|
---|
585 | {
|
---|
586 | PGMM pGMM;
|
---|
587 | /** The index of the chunk mutex. */
|
---|
588 | uint8_t iChunkMtx;
|
---|
589 | /** The relevant flags (GMMR0CHUNK_MTX_XXX). */
|
---|
590 | uint8_t fFlags;
|
---|
591 | } GMMR0CHUNKMTXSTATE;
|
---|
592 | /** Pointer to a chunk mutex state. */
|
---|
593 | typedef GMMR0CHUNKMTXSTATE *PGMMR0CHUNKMTXSTATE;
|
---|
594 |
|
---|
595 | /** @name GMMR0CHUNK_MTX_XXX
|
---|
596 | * @{ */
|
---|
597 | #define GMMR0CHUNK_MTX_INVALID UINT32_C(0)
|
---|
598 | #define GMMR0CHUNK_MTX_KEEP_GIANT UINT32_C(1)
|
---|
599 | #define GMMR0CHUNK_MTX_RETAKE_GIANT UINT32_C(2)
|
---|
600 | #define GMMR0CHUNK_MTX_DROP_GIANT UINT32_C(3)
|
---|
601 | #define GMMR0CHUNK_MTX_END UINT32_C(4)
|
---|
602 | /** @} */
|
---|
603 |
|
---|
604 |
|
---|
605 | /** The maximum number of shared modules per-vm. */
|
---|
606 | #define GMM_MAX_SHARED_PER_VM_MODULES 2048
|
---|
607 | /** The maximum number of shared modules GMM is allowed to track. */
|
---|
608 | #define GMM_MAX_SHARED_GLOBAL_MODULES 16834
|
---|
609 |
|
---|
610 |
|
---|
611 | /**
|
---|
612 | * Argument packet for gmmR0SharedModuleCleanup.
|
---|
613 | */
|
---|
614 | typedef struct GMMR0SHMODPERVMDTORARGS
|
---|
615 | {
|
---|
616 | PGVM pGVM;
|
---|
617 | PGMM pGMM;
|
---|
618 | } GMMR0SHMODPERVMDTORARGS;
|
---|
619 |
|
---|
620 | /**
|
---|
621 | * Argument packet for gmmR0CheckSharedModule.
|
---|
622 | */
|
---|
623 | typedef struct GMMCHECKSHAREDMODULEINFO
|
---|
624 | {
|
---|
625 | PGVM pGVM;
|
---|
626 | VMCPUID idCpu;
|
---|
627 | } GMMCHECKSHAREDMODULEINFO;
|
---|
628 |
|
---|
629 |
|
---|
630 | /*********************************************************************************************************************************
|
---|
631 | * Global Variables *
|
---|
632 | *********************************************************************************************************************************/
|
---|
633 | /** Pointer to the GMM instance data. */
|
---|
634 | static PGMM g_pGMM = NULL;
|
---|
635 |
|
---|
636 | /** Macro for obtaining and validating the g_pGMM pointer.
|
---|
637 | *
|
---|
638 | * On failure it will return from the invoking function with the specified
|
---|
639 | * return value.
|
---|
640 | *
|
---|
641 | * @param pGMM The name of the pGMM variable.
|
---|
642 | * @param rc The return value on failure. Use VERR_GMM_INSTANCE for VBox
|
---|
643 | * status codes.
|
---|
644 | */
|
---|
645 | #define GMM_GET_VALID_INSTANCE(pGMM, rc) \
|
---|
646 | do { \
|
---|
647 | (pGMM) = g_pGMM; \
|
---|
648 | AssertPtrReturn((pGMM), (rc)); \
|
---|
649 | AssertMsgReturn((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic), (rc)); \
|
---|
650 | } while (0)
|
---|
651 |
|
---|
652 | /** Macro for obtaining and validating the g_pGMM pointer, void function
|
---|
653 | * variant.
|
---|
654 | *
|
---|
655 | * On failure it will return from the invoking function.
|
---|
656 | *
|
---|
657 | * @param pGMM The name of the pGMM variable.
|
---|
658 | */
|
---|
659 | #define GMM_GET_VALID_INSTANCE_VOID(pGMM) \
|
---|
660 | do { \
|
---|
661 | (pGMM) = g_pGMM; \
|
---|
662 | AssertPtrReturnVoid((pGMM)); \
|
---|
663 | AssertMsgReturnVoid((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic)); \
|
---|
664 | } while (0)
|
---|
665 |
|
---|
666 |
|
---|
667 | /** @def GMM_CHECK_SANITY_UPON_ENTERING
|
---|
668 | * Checks the sanity of the GMM instance data before making changes.
|
---|
669 | *
|
---|
670 | * This is macro is a stub by default and must be enabled manually in the code.
|
---|
671 | *
|
---|
672 | * @returns true if sane, false if not.
|
---|
673 | * @param pGMM The name of the pGMM variable.
|
---|
674 | */
|
---|
675 | #if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0
|
---|
676 | # define GMM_CHECK_SANITY_UPON_ENTERING(pGMM) (RT_LIKELY(gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0))
|
---|
677 | #else
|
---|
678 | # define GMM_CHECK_SANITY_UPON_ENTERING(pGMM) (true)
|
---|
679 | #endif
|
---|
680 |
|
---|
681 | /** @def GMM_CHECK_SANITY_UPON_LEAVING
|
---|
682 | * Checks the sanity of the GMM instance data after making changes.
|
---|
683 | *
|
---|
684 | * This is macro is a stub by default and must be enabled manually in the code.
|
---|
685 | *
|
---|
686 | * @returns true if sane, false if not.
|
---|
687 | * @param pGMM The name of the pGMM variable.
|
---|
688 | */
|
---|
689 | #if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0
|
---|
690 | # define GMM_CHECK_SANITY_UPON_LEAVING(pGMM) (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
|
---|
691 | #else
|
---|
692 | # define GMM_CHECK_SANITY_UPON_LEAVING(pGMM) (true)
|
---|
693 | #endif
|
---|
694 |
|
---|
695 | /** @def GMM_CHECK_SANITY_IN_LOOPS
|
---|
696 | * Checks the sanity of the GMM instance in the allocation loops.
|
---|
697 | *
|
---|
698 | * This is macro is a stub by default and must be enabled manually in the code.
|
---|
699 | *
|
---|
700 | * @returns true if sane, false if not.
|
---|
701 | * @param pGMM The name of the pGMM variable.
|
---|
702 | */
|
---|
703 | #if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0
|
---|
704 | # define GMM_CHECK_SANITY_IN_LOOPS(pGMM) (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
|
---|
705 | #else
|
---|
706 | # define GMM_CHECK_SANITY_IN_LOOPS(pGMM) (true)
|
---|
707 | #endif
|
---|
708 |
|
---|
709 |
|
---|
710 | /*********************************************************************************************************************************
|
---|
711 | * Internal Functions *
|
---|
712 | *********************************************************************************************************************************/
|
---|
713 | static DECLCALLBACK(int) gmmR0TermDestroyChunk(PAVLU32NODECORE pNode, void *pvGMM);
|
---|
714 | static bool gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
|
---|
715 | DECLINLINE(void) gmmR0UnlinkChunk(PGMMCHUNK pChunk);
|
---|
716 | DECLINLINE(void) gmmR0LinkChunk(PGMMCHUNK pChunk, PGMMCHUNKFREESET pSet);
|
---|
717 | DECLINLINE(void) gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
|
---|
718 | #ifdef GMMR0_WITH_SANITY_CHECK
|
---|
719 | static uint32_t gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo);
|
---|
720 | #endif
|
---|
721 | static bool gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem);
|
---|
722 | DECLINLINE(void) gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage);
|
---|
723 | DECLINLINE(void) gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage);
|
---|
724 | static int gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
|
---|
725 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
726 | static void gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM);
|
---|
727 | # ifdef VBOX_STRICT
|
---|
728 | static uint32_t gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage);
|
---|
729 | # endif
|
---|
730 | #endif
|
---|
731 |
|
---|
732 |
|
---|
733 |
|
---|
734 | /**
|
---|
735 | * Initializes the GMM component.
|
---|
736 | *
|
---|
737 | * This is called when the VMMR0.r0 module is loaded and protected by the
|
---|
738 | * loader semaphore.
|
---|
739 | *
|
---|
740 | * @returns VBox status code.
|
---|
741 | */
|
---|
742 | GMMR0DECL(int) GMMR0Init(void)
|
---|
743 | {
|
---|
744 | LogFlow(("GMMInit:\n"));
|
---|
745 |
|
---|
746 | /* Currently assuming same host and guest page size here. Can change it to
|
---|
747 | dish out guest pages with different size from the host page later if
|
---|
748 | needed, though a restriction would be the host page size must be larger
|
---|
749 | than the guest page size. */
|
---|
750 | AssertCompile(GUEST_PAGE_SIZE == HOST_PAGE_SIZE);
|
---|
751 | AssertCompile(GUEST_PAGE_SIZE <= HOST_PAGE_SIZE);
|
---|
752 |
|
---|
753 | /*
|
---|
754 | * Allocate the instance data and the locks.
|
---|
755 | */
|
---|
756 | PGMM pGMM = (PGMM)RTMemAllocZ(sizeof(*pGMM));
|
---|
757 | if (!pGMM)
|
---|
758 | return VERR_NO_MEMORY;
|
---|
759 |
|
---|
760 | pGMM->u32Magic = GMM_MAGIC;
|
---|
761 | for (unsigned i = 0; i < RT_ELEMENTS(pGMM->ChunkTLB.aEntries); i++)
|
---|
762 | pGMM->ChunkTLB.aEntries[i].idChunk = NIL_GMM_CHUNKID;
|
---|
763 | RTListInit(&pGMM->ChunkList);
|
---|
764 | ASMBitSet(&pGMM->bmChunkId[0], NIL_GMM_CHUNKID);
|
---|
765 |
|
---|
766 | #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
767 | int rc = RTCritSectInit(&pGMM->GiantCritSect);
|
---|
768 | #else
|
---|
769 | int rc = RTSemFastMutexCreate(&pGMM->hMtx);
|
---|
770 | #endif
|
---|
771 | if (RT_SUCCESS(rc))
|
---|
772 | {
|
---|
773 | unsigned iMtx;
|
---|
774 | for (iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++)
|
---|
775 | {
|
---|
776 | rc = RTSemFastMutexCreate(&pGMM->aChunkMtx[iMtx].hMtx);
|
---|
777 | if (RT_FAILURE(rc))
|
---|
778 | break;
|
---|
779 | }
|
---|
780 | pGMM->hSpinLockTree = NIL_RTSPINLOCK;
|
---|
781 | if (RT_SUCCESS(rc))
|
---|
782 | rc = RTSpinlockCreate(&pGMM->hSpinLockTree, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "gmm-chunk-tree");
|
---|
783 | pGMM->hSpinLockChunkId = NIL_RTSPINLOCK;
|
---|
784 | if (RT_SUCCESS(rc))
|
---|
785 | rc = RTSpinlockCreate(&pGMM->hSpinLockChunkId, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "gmm-chunk-id");
|
---|
786 | if (RT_SUCCESS(rc))
|
---|
787 | {
|
---|
788 | /*
|
---|
789 | * Figure out how we're going to allocate stuff (only applicable to
|
---|
790 | * host with linear physical memory mappings).
|
---|
791 | */
|
---|
792 | pGMM->fBoundMemoryMode = false;
|
---|
793 | #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
794 | pGMM->fHasWorkingAllocPhysNC = false;
|
---|
795 |
|
---|
796 | RTR0MEMOBJ hMemObj;
|
---|
797 | rc = RTR0MemObjAllocPhysNC(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS);
|
---|
798 | if (RT_SUCCESS(rc))
|
---|
799 | {
|
---|
800 | rc = RTR0MemObjFree(hMemObj, true);
|
---|
801 | AssertRC(rc);
|
---|
802 | pGMM->fHasWorkingAllocPhysNC = true;
|
---|
803 | }
|
---|
804 | else if (rc != VERR_NOT_SUPPORTED)
|
---|
805 | SUPR0Printf("GMMR0Init: Warning! RTR0MemObjAllocPhysNC(, %u, NIL_RTHCPHYS) -> %d!\n", GMM_CHUNK_SIZE, rc);
|
---|
806 | # endif
|
---|
807 |
|
---|
808 | /*
|
---|
809 | * Query system page count and guess a reasonable cMaxPages value.
|
---|
810 | */
|
---|
811 | pGMM->cMaxPages = UINT32_MAX; /** @todo IPRT function for query ram size and such. */
|
---|
812 |
|
---|
813 | /*
|
---|
814 | * The idFreeGeneration value should be set so we actually trigger the
|
---|
815 | * wrap-around invalidation handling during a typical test run.
|
---|
816 | */
|
---|
817 | pGMM->idFreeGeneration = UINT64_MAX / 4 - 128;
|
---|
818 |
|
---|
819 | g_pGMM = pGMM;
|
---|
820 | #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
821 | LogFlow(("GMMInit: pGMM=%p fBoundMemoryMode=%RTbool fHasWorkingAllocPhysNC=%RTbool\n", pGMM, pGMM->fBoundMemoryMode, pGMM->fHasWorkingAllocPhysNC));
|
---|
822 | #else
|
---|
823 | LogFlow(("GMMInit: pGMM=%p fBoundMemoryMode=%RTbool\n", pGMM, pGMM->fBoundMemoryMode));
|
---|
824 | #endif
|
---|
825 | return VINF_SUCCESS;
|
---|
826 | }
|
---|
827 |
|
---|
828 | /*
|
---|
829 | * Bail out.
|
---|
830 | */
|
---|
831 | RTSpinlockDestroy(pGMM->hSpinLockChunkId);
|
---|
832 | RTSpinlockDestroy(pGMM->hSpinLockTree);
|
---|
833 | while (iMtx-- > 0)
|
---|
834 | RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx);
|
---|
835 | #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
836 | RTCritSectDelete(&pGMM->GiantCritSect);
|
---|
837 | #else
|
---|
838 | RTSemFastMutexDestroy(pGMM->hMtx);
|
---|
839 | #endif
|
---|
840 | }
|
---|
841 |
|
---|
842 | pGMM->u32Magic = 0;
|
---|
843 | RTMemFree(pGMM);
|
---|
844 | SUPR0Printf("GMMR0Init: failed! rc=%d\n", rc);
|
---|
845 | return rc;
|
---|
846 | }
|
---|
847 |
|
---|
848 |
|
---|
849 | /**
|
---|
850 | * Terminates the GMM component.
|
---|
851 | */
|
---|
852 | GMMR0DECL(void) GMMR0Term(void)
|
---|
853 | {
|
---|
854 | LogFlow(("GMMTerm:\n"));
|
---|
855 |
|
---|
856 | /*
|
---|
857 | * Take care / be paranoid...
|
---|
858 | */
|
---|
859 | PGMM pGMM = g_pGMM;
|
---|
860 | if (!RT_VALID_PTR(pGMM))
|
---|
861 | return;
|
---|
862 | if (pGMM->u32Magic != GMM_MAGIC)
|
---|
863 | {
|
---|
864 | SUPR0Printf("GMMR0Term: u32Magic=%#x\n", pGMM->u32Magic);
|
---|
865 | return;
|
---|
866 | }
|
---|
867 |
|
---|
868 | /*
|
---|
869 | * Undo what init did and free all the resources we've acquired.
|
---|
870 | */
|
---|
871 | /* Destroy the fundamentals. */
|
---|
872 | g_pGMM = NULL;
|
---|
873 | pGMM->u32Magic = ~GMM_MAGIC;
|
---|
874 | #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
875 | RTCritSectDelete(&pGMM->GiantCritSect);
|
---|
876 | #else
|
---|
877 | RTSemFastMutexDestroy(pGMM->hMtx);
|
---|
878 | pGMM->hMtx = NIL_RTSEMFASTMUTEX;
|
---|
879 | #endif
|
---|
880 | RTSpinlockDestroy(pGMM->hSpinLockTree);
|
---|
881 | pGMM->hSpinLockTree = NIL_RTSPINLOCK;
|
---|
882 | RTSpinlockDestroy(pGMM->hSpinLockChunkId);
|
---|
883 | pGMM->hSpinLockChunkId = NIL_RTSPINLOCK;
|
---|
884 |
|
---|
885 | /* Free any chunks still hanging around. */
|
---|
886 | RTAvlU32Destroy(&pGMM->pChunks, gmmR0TermDestroyChunk, pGMM);
|
---|
887 |
|
---|
888 | /* Destroy the chunk locks. */
|
---|
889 | for (unsigned iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++)
|
---|
890 | {
|
---|
891 | Assert(pGMM->aChunkMtx[iMtx].cUsers == 0);
|
---|
892 | RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx);
|
---|
893 | pGMM->aChunkMtx[iMtx].hMtx = NIL_RTSEMFASTMUTEX;
|
---|
894 | }
|
---|
895 |
|
---|
896 | /* Finally the instance data itself. */
|
---|
897 | RTMemFree(pGMM);
|
---|
898 | LogFlow(("GMMTerm: done\n"));
|
---|
899 | }
|
---|
900 |
|
---|
901 |
|
---|
902 | /**
|
---|
903 | * RTAvlU32Destroy callback.
|
---|
904 | *
|
---|
905 | * @returns 0
|
---|
906 | * @param pNode The node to destroy.
|
---|
907 | * @param pvGMM The GMM handle.
|
---|
908 | */
|
---|
909 | static DECLCALLBACK(int) gmmR0TermDestroyChunk(PAVLU32NODECORE pNode, void *pvGMM)
|
---|
910 | {
|
---|
911 | PGMMCHUNK pChunk = (PGMMCHUNK)pNode;
|
---|
912 |
|
---|
913 | if (pChunk->cFree != GMM_CHUNK_NUM_PAGES)
|
---|
914 | SUPR0Printf("GMMR0Term: %RKv/%#x: cFree=%d cPrivate=%d cShared=%d cMappings=%d\n", pChunk,
|
---|
915 | pChunk->Core.Key, pChunk->cFree, pChunk->cPrivate, pChunk->cShared, pChunk->cMappingsX);
|
---|
916 |
|
---|
917 | int rc = RTR0MemObjFree(pChunk->hMemObj, true /* fFreeMappings */);
|
---|
918 | if (RT_FAILURE(rc))
|
---|
919 | {
|
---|
920 | SUPR0Printf("GMMR0Term: %RKv/%#x: RTRMemObjFree(%RKv,true) -> %d (cMappings=%d)\n", pChunk,
|
---|
921 | pChunk->Core.Key, pChunk->hMemObj, rc, pChunk->cMappingsX);
|
---|
922 | AssertRC(rc);
|
---|
923 | }
|
---|
924 | pChunk->hMemObj = NIL_RTR0MEMOBJ;
|
---|
925 |
|
---|
926 | RTMemFree(pChunk->paMappingsX);
|
---|
927 | pChunk->paMappingsX = NULL;
|
---|
928 |
|
---|
929 | RTMemFree(pChunk);
|
---|
930 | NOREF(pvGMM);
|
---|
931 | return 0;
|
---|
932 | }
|
---|
933 |
|
---|
934 |
|
---|
935 | /**
|
---|
936 | * Initializes the per-VM data for the GMM.
|
---|
937 | *
|
---|
938 | * This is called from within the GVMM lock (from GVMMR0CreateVM)
|
---|
939 | * and should only initialize the data members so GMMR0CleanupVM
|
---|
940 | * can deal with them. We reserve no memory or anything here,
|
---|
941 | * that's done later in GMMR0InitVM.
|
---|
942 | *
|
---|
943 | * @param pGVM Pointer to the Global VM structure.
|
---|
944 | */
|
---|
945 | GMMR0DECL(int) GMMR0InitPerVMData(PGVM pGVM)
|
---|
946 | {
|
---|
947 | AssertCompile(RT_SIZEOFMEMB(GVM,gmm.s) <= RT_SIZEOFMEMB(GVM,gmm.padding));
|
---|
948 |
|
---|
949 | pGVM->gmm.s.Stats.enmPolicy = GMMOCPOLICY_INVALID;
|
---|
950 | pGVM->gmm.s.Stats.enmPriority = GMMPRIORITY_INVALID;
|
---|
951 | pGVM->gmm.s.Stats.fMayAllocate = false;
|
---|
952 |
|
---|
953 | pGVM->gmm.s.hChunkTlbSpinLock = NIL_RTSPINLOCK;
|
---|
954 | int rc = RTSpinlockCreate(&pGVM->gmm.s.hChunkTlbSpinLock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "per-vm-chunk-tlb");
|
---|
955 | AssertRCReturn(rc, rc);
|
---|
956 |
|
---|
957 | return VINF_SUCCESS;
|
---|
958 | }
|
---|
959 |
|
---|
960 |
|
---|
961 | /**
|
---|
962 | * Acquires the GMM giant lock.
|
---|
963 | *
|
---|
964 | * @returns Assert status code from RTSemFastMutexRequest.
|
---|
965 | * @param pGMM Pointer to the GMM instance.
|
---|
966 | */
|
---|
967 | static int gmmR0MutexAcquire(PGMM pGMM)
|
---|
968 | {
|
---|
969 | ASMAtomicIncU32(&pGMM->cMtxContenders);
|
---|
970 | #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
971 | int rc = RTCritSectEnter(&pGMM->GiantCritSect);
|
---|
972 | #else
|
---|
973 | int rc = RTSemFastMutexRequest(pGMM->hMtx);
|
---|
974 | #endif
|
---|
975 | ASMAtomicDecU32(&pGMM->cMtxContenders);
|
---|
976 | AssertRC(rc);
|
---|
977 | #ifdef VBOX_STRICT
|
---|
978 | pGMM->hMtxOwner = RTThreadNativeSelf();
|
---|
979 | #endif
|
---|
980 | return rc;
|
---|
981 | }
|
---|
982 |
|
---|
983 |
|
---|
984 | /**
|
---|
985 | * Releases the GMM giant lock.
|
---|
986 | *
|
---|
987 | * @returns Assert status code from RTSemFastMutexRequest.
|
---|
988 | * @param pGMM Pointer to the GMM instance.
|
---|
989 | */
|
---|
990 | static int gmmR0MutexRelease(PGMM pGMM)
|
---|
991 | {
|
---|
992 | #ifdef VBOX_STRICT
|
---|
993 | pGMM->hMtxOwner = NIL_RTNATIVETHREAD;
|
---|
994 | #endif
|
---|
995 | #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
996 | int rc = RTCritSectLeave(&pGMM->GiantCritSect);
|
---|
997 | #else
|
---|
998 | int rc = RTSemFastMutexRelease(pGMM->hMtx);
|
---|
999 | AssertRC(rc);
|
---|
1000 | #endif
|
---|
1001 | return rc;
|
---|
1002 | }
|
---|
1003 |
|
---|
1004 |
|
---|
1005 | /**
|
---|
1006 | * Yields the GMM giant lock if there is contention and a certain minimum time
|
---|
1007 | * has elapsed since we took it.
|
---|
1008 | *
|
---|
1009 | * @returns @c true if the mutex was yielded, @c false if not.
|
---|
1010 | * @param pGMM Pointer to the GMM instance.
|
---|
1011 | * @param puLockNanoTS Where the lock acquisition time stamp is kept
|
---|
1012 | * (in/out).
|
---|
1013 | */
|
---|
1014 | static bool gmmR0MutexYield(PGMM pGMM, uint64_t *puLockNanoTS)
|
---|
1015 | {
|
---|
1016 | /*
|
---|
1017 | * If nobody is contending the mutex, don't bother checking the time.
|
---|
1018 | */
|
---|
1019 | if (ASMAtomicReadU32(&pGMM->cMtxContenders) == 0)
|
---|
1020 | return false;
|
---|
1021 |
|
---|
1022 | /*
|
---|
1023 | * Don't yield if we haven't executed for at least 2 milliseconds.
|
---|
1024 | */
|
---|
1025 | uint64_t uNanoNow = RTTimeSystemNanoTS();
|
---|
1026 | if (uNanoNow - *puLockNanoTS < UINT32_C(2000000))
|
---|
1027 | return false;
|
---|
1028 |
|
---|
1029 | /*
|
---|
1030 | * Yield the mutex.
|
---|
1031 | */
|
---|
1032 | #ifdef VBOX_STRICT
|
---|
1033 | pGMM->hMtxOwner = NIL_RTNATIVETHREAD;
|
---|
1034 | #endif
|
---|
1035 | ASMAtomicIncU32(&pGMM->cMtxContenders);
|
---|
1036 | #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
1037 | int rc1 = RTCritSectLeave(&pGMM->GiantCritSect); AssertRC(rc1);
|
---|
1038 | #else
|
---|
1039 | int rc1 = RTSemFastMutexRelease(pGMM->hMtx); AssertRC(rc1);
|
---|
1040 | #endif
|
---|
1041 |
|
---|
1042 | RTThreadYield();
|
---|
1043 |
|
---|
1044 | #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
|
---|
1045 | int rc2 = RTCritSectEnter(&pGMM->GiantCritSect); AssertRC(rc2);
|
---|
1046 | #else
|
---|
1047 | int rc2 = RTSemFastMutexRequest(pGMM->hMtx); AssertRC(rc2);
|
---|
1048 | #endif
|
---|
1049 | *puLockNanoTS = RTTimeSystemNanoTS();
|
---|
1050 | ASMAtomicDecU32(&pGMM->cMtxContenders);
|
---|
1051 | #ifdef VBOX_STRICT
|
---|
1052 | pGMM->hMtxOwner = RTThreadNativeSelf();
|
---|
1053 | #endif
|
---|
1054 |
|
---|
1055 | return true;
|
---|
1056 | }
|
---|
1057 |
|
---|
1058 |
|
---|
1059 | /**
|
---|
1060 | * Acquires a chunk lock.
|
---|
1061 | *
|
---|
1062 | * The caller must own the giant lock.
|
---|
1063 | *
|
---|
1064 | * @returns Assert status code from RTSemFastMutexRequest.
|
---|
1065 | * @param pMtxState The chunk mutex state info. (Avoids
|
---|
1066 | * passing the same flags and stuff around
|
---|
1067 | * for subsequent release and drop-giant
|
---|
1068 | * calls.)
|
---|
1069 | * @param pGMM Pointer to the GMM instance.
|
---|
1070 | * @param pChunk Pointer to the chunk.
|
---|
1071 | * @param fFlags Flags regarding the giant lock, GMMR0CHUNK_MTX_XXX.
|
---|
1072 | */
|
---|
1073 | static int gmmR0ChunkMutexAcquire(PGMMR0CHUNKMTXSTATE pMtxState, PGMM pGMM, PGMMCHUNK pChunk, uint32_t fFlags)
|
---|
1074 | {
|
---|
1075 | Assert(fFlags > GMMR0CHUNK_MTX_INVALID && fFlags < GMMR0CHUNK_MTX_END);
|
---|
1076 | Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
|
---|
1077 |
|
---|
1078 | pMtxState->pGMM = pGMM;
|
---|
1079 | pMtxState->fFlags = (uint8_t)fFlags;
|
---|
1080 |
|
---|
1081 | /*
|
---|
1082 | * Get the lock index and reference the lock.
|
---|
1083 | */
|
---|
1084 | Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
|
---|
1085 | uint32_t iChunkMtx = pChunk->iChunkMtx;
|
---|
1086 | if (iChunkMtx == UINT8_MAX)
|
---|
1087 | {
|
---|
1088 | iChunkMtx = pGMM->iNextChunkMtx++;
|
---|
1089 | iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
|
---|
1090 |
|
---|
1091 | /* Try get an unused one... */
|
---|
1092 | if (pGMM->aChunkMtx[iChunkMtx].cUsers)
|
---|
1093 | {
|
---|
1094 | iChunkMtx = pGMM->iNextChunkMtx++;
|
---|
1095 | iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
|
---|
1096 | if (pGMM->aChunkMtx[iChunkMtx].cUsers)
|
---|
1097 | {
|
---|
1098 | iChunkMtx = pGMM->iNextChunkMtx++;
|
---|
1099 | iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
|
---|
1100 | if (pGMM->aChunkMtx[iChunkMtx].cUsers)
|
---|
1101 | {
|
---|
1102 | iChunkMtx = pGMM->iNextChunkMtx++;
|
---|
1103 | iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
|
---|
1104 | }
|
---|
1105 | }
|
---|
1106 | }
|
---|
1107 |
|
---|
1108 | pChunk->iChunkMtx = iChunkMtx;
|
---|
1109 | }
|
---|
1110 | AssertCompile(RT_ELEMENTS(pGMM->aChunkMtx) < UINT8_MAX);
|
---|
1111 | pMtxState->iChunkMtx = (uint8_t)iChunkMtx;
|
---|
1112 | ASMAtomicIncU32(&pGMM->aChunkMtx[iChunkMtx].cUsers);
|
---|
1113 |
|
---|
1114 | /*
|
---|
1115 | * Drop the giant?
|
---|
1116 | */
|
---|
1117 | if (fFlags != GMMR0CHUNK_MTX_KEEP_GIANT)
|
---|
1118 | {
|
---|
1119 | /** @todo GMM life cycle cleanup (we may race someone
|
---|
1120 | * destroying and cleaning up GMM)? */
|
---|
1121 | gmmR0MutexRelease(pGMM);
|
---|
1122 | }
|
---|
1123 |
|
---|
1124 | /*
|
---|
1125 | * Take the chunk mutex.
|
---|
1126 | */
|
---|
1127 | int rc = RTSemFastMutexRequest(pGMM->aChunkMtx[iChunkMtx].hMtx);
|
---|
1128 | AssertRC(rc);
|
---|
1129 | return rc;
|
---|
1130 | }
|
---|
1131 |
|
---|
1132 |
|
---|
1133 | /**
|
---|
1134 | * Releases the GMM giant lock.
|
---|
1135 | *
|
---|
1136 | * @returns Assert status code from RTSemFastMutexRequest.
|
---|
1137 | * @param pMtxState Pointer to the chunk mutex state.
|
---|
1138 | * @param pChunk Pointer to the chunk if it's still
|
---|
1139 | * alive, NULL if it isn't. This is used to deassociate
|
---|
1140 | * the chunk from the mutex on the way out so a new one
|
---|
1141 | * can be selected next time, thus avoiding contented
|
---|
1142 | * mutexes.
|
---|
1143 | */
|
---|
1144 | static int gmmR0ChunkMutexRelease(PGMMR0CHUNKMTXSTATE pMtxState, PGMMCHUNK pChunk)
|
---|
1145 | {
|
---|
1146 | PGMM pGMM = pMtxState->pGMM;
|
---|
1147 |
|
---|
1148 | /*
|
---|
1149 | * Release the chunk mutex and reacquire the giant if requested.
|
---|
1150 | */
|
---|
1151 | int rc = RTSemFastMutexRelease(pGMM->aChunkMtx[pMtxState->iChunkMtx].hMtx);
|
---|
1152 | AssertRC(rc);
|
---|
1153 | if (pMtxState->fFlags == GMMR0CHUNK_MTX_RETAKE_GIANT)
|
---|
1154 | rc = gmmR0MutexAcquire(pGMM);
|
---|
1155 | else
|
---|
1156 | Assert((pMtxState->fFlags != GMMR0CHUNK_MTX_DROP_GIANT) == (pGMM->hMtxOwner == RTThreadNativeSelf()));
|
---|
1157 |
|
---|
1158 | /*
|
---|
1159 | * Drop the chunk mutex user reference and deassociate it from the chunk
|
---|
1160 | * when possible.
|
---|
1161 | */
|
---|
1162 | if ( ASMAtomicDecU32(&pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers) == 0
|
---|
1163 | && pChunk
|
---|
1164 | && RT_SUCCESS(rc) )
|
---|
1165 | {
|
---|
1166 | if (pMtxState->fFlags != GMMR0CHUNK_MTX_DROP_GIANT)
|
---|
1167 | pChunk->iChunkMtx = UINT8_MAX;
|
---|
1168 | else
|
---|
1169 | {
|
---|
1170 | rc = gmmR0MutexAcquire(pGMM);
|
---|
1171 | if (RT_SUCCESS(rc))
|
---|
1172 | {
|
---|
1173 | if (pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers == 0)
|
---|
1174 | pChunk->iChunkMtx = UINT8_MAX;
|
---|
1175 | rc = gmmR0MutexRelease(pGMM);
|
---|
1176 | }
|
---|
1177 | }
|
---|
1178 | }
|
---|
1179 |
|
---|
1180 | pMtxState->pGMM = NULL;
|
---|
1181 | return rc;
|
---|
1182 | }
|
---|
1183 |
|
---|
1184 |
|
---|
1185 | /**
|
---|
1186 | * Drops the giant GMM lock we kept in gmmR0ChunkMutexAcquire while keeping the
|
---|
1187 | * chunk locked.
|
---|
1188 | *
|
---|
1189 | * This only works if gmmR0ChunkMutexAcquire was called with
|
---|
1190 | * GMMR0CHUNK_MTX_KEEP_GIANT. gmmR0ChunkMutexRelease will retake the giant
|
---|
1191 | * mutex, i.e. behave as if GMMR0CHUNK_MTX_RETAKE_GIANT was used.
|
---|
1192 | *
|
---|
1193 | * @returns VBox status code (assuming success is ok).
|
---|
1194 | * @param pMtxState Pointer to the chunk mutex state.
|
---|
1195 | */
|
---|
1196 | static int gmmR0ChunkMutexDropGiant(PGMMR0CHUNKMTXSTATE pMtxState)
|
---|
1197 | {
|
---|
1198 | AssertReturn(pMtxState->fFlags == GMMR0CHUNK_MTX_KEEP_GIANT, VERR_GMM_MTX_FLAGS);
|
---|
1199 | Assert(pMtxState->pGMM->hMtxOwner == RTThreadNativeSelf());
|
---|
1200 | pMtxState->fFlags = GMMR0CHUNK_MTX_RETAKE_GIANT;
|
---|
1201 | /** @todo GMM life cycle cleanup (we may race someone
|
---|
1202 | * destroying and cleaning up GMM)? */
|
---|
1203 | return gmmR0MutexRelease(pMtxState->pGMM);
|
---|
1204 | }
|
---|
1205 |
|
---|
1206 |
|
---|
1207 | /**
|
---|
1208 | * For experimenting with NUMA affinity and such.
|
---|
1209 | *
|
---|
1210 | * @returns The current NUMA Node ID.
|
---|
1211 | */
|
---|
1212 | static uint16_t gmmR0GetCurrentNumaNodeId(void)
|
---|
1213 | {
|
---|
1214 | #if 1
|
---|
1215 | return GMM_CHUNK_NUMA_ID_UNKNOWN;
|
---|
1216 | #else
|
---|
1217 | return RTMpCpuId() / 16;
|
---|
1218 | #endif
|
---|
1219 | }
|
---|
1220 |
|
---|
1221 |
|
---|
1222 |
|
---|
1223 | /**
|
---|
1224 | * Cleans up when a VM is terminating.
|
---|
1225 | *
|
---|
1226 | * @param pGVM Pointer to the Global VM structure.
|
---|
1227 | */
|
---|
1228 | GMMR0DECL(void) GMMR0CleanupVM(PGVM pGVM)
|
---|
1229 | {
|
---|
1230 | LogFlow(("GMMR0CleanupVM: pGVM=%p:{.hSelf=%#x}\n", pGVM, pGVM->hSelf));
|
---|
1231 |
|
---|
1232 | PGMM pGMM;
|
---|
1233 | GMM_GET_VALID_INSTANCE_VOID(pGMM);
|
---|
1234 |
|
---|
1235 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
1236 | /*
|
---|
1237 | * Clean up all registered shared modules first.
|
---|
1238 | */
|
---|
1239 | gmmR0SharedModuleCleanup(pGMM, pGVM);
|
---|
1240 | #endif
|
---|
1241 |
|
---|
1242 | gmmR0MutexAcquire(pGMM);
|
---|
1243 | uint64_t uLockNanoTS = RTTimeSystemNanoTS();
|
---|
1244 | GMM_CHECK_SANITY_UPON_ENTERING(pGMM);
|
---|
1245 |
|
---|
1246 | /*
|
---|
1247 | * The policy is 'INVALID' until the initial reservation
|
---|
1248 | * request has been serviced.
|
---|
1249 | */
|
---|
1250 | if ( pGVM->gmm.s.Stats.enmPolicy > GMMOCPOLICY_INVALID
|
---|
1251 | && pGVM->gmm.s.Stats.enmPolicy < GMMOCPOLICY_END)
|
---|
1252 | {
|
---|
1253 | /*
|
---|
1254 | * If it's the last VM around, we can skip walking all the chunk looking
|
---|
1255 | * for the pages owned by this VM and instead flush the whole shebang.
|
---|
1256 | *
|
---|
1257 | * This takes care of the eventuality that a VM has left shared page
|
---|
1258 | * references behind (shouldn't happen of course, but you never know).
|
---|
1259 | */
|
---|
1260 | Assert(pGMM->cRegisteredVMs);
|
---|
1261 | pGMM->cRegisteredVMs--;
|
---|
1262 |
|
---|
1263 | /*
|
---|
1264 | * Walk the entire pool looking for pages that belong to this VM
|
---|
1265 | * and leftover mappings. (This'll only catch private pages,
|
---|
1266 | * shared pages will be 'left behind'.)
|
---|
1267 | */
|
---|
1268 | /** @todo r=bird: This scanning+freeing could be optimized in bound mode! */
|
---|
1269 | uint64_t cPrivatePages = pGVM->gmm.s.Stats.cPrivatePages; /* save */
|
---|
1270 |
|
---|
1271 | unsigned iCountDown = 64;
|
---|
1272 | bool fRedoFromStart;
|
---|
1273 | PGMMCHUNK pChunk;
|
---|
1274 | do
|
---|
1275 | {
|
---|
1276 | fRedoFromStart = false;
|
---|
1277 | RTListForEachReverse(&pGMM->ChunkList, pChunk, GMMCHUNK, ListNode)
|
---|
1278 | {
|
---|
1279 | uint32_t const cFreeChunksOld = pGMM->cFreedChunks;
|
---|
1280 | if ( ( !pGMM->fBoundMemoryMode
|
---|
1281 | || pChunk->hGVM == pGVM->hSelf)
|
---|
1282 | && gmmR0CleanupVMScanChunk(pGMM, pGVM, pChunk))
|
---|
1283 | {
|
---|
1284 | /* We left the giant mutex, so reset the yield counters. */
|
---|
1285 | uLockNanoTS = RTTimeSystemNanoTS();
|
---|
1286 | iCountDown = 64;
|
---|
1287 | }
|
---|
1288 | else
|
---|
1289 | {
|
---|
1290 | /* Didn't leave it, so do normal yielding. */
|
---|
1291 | if (!iCountDown)
|
---|
1292 | gmmR0MutexYield(pGMM, &uLockNanoTS);
|
---|
1293 | else
|
---|
1294 | iCountDown--;
|
---|
1295 | }
|
---|
1296 | if (pGMM->cFreedChunks != cFreeChunksOld)
|
---|
1297 | {
|
---|
1298 | fRedoFromStart = true;
|
---|
1299 | break;
|
---|
1300 | }
|
---|
1301 | }
|
---|
1302 | } while (fRedoFromStart);
|
---|
1303 |
|
---|
1304 | if (pGVM->gmm.s.Stats.cPrivatePages)
|
---|
1305 | SUPR0Printf("GMMR0CleanupVM: hGVM=%#x has %#x private pages that cannot be found!\n", pGVM->hSelf, pGVM->gmm.s.Stats.cPrivatePages);
|
---|
1306 |
|
---|
1307 | pGMM->cAllocatedPages -= cPrivatePages;
|
---|
1308 |
|
---|
1309 | /*
|
---|
1310 | * Free empty chunks.
|
---|
1311 | */
|
---|
1312 | PGMMCHUNKFREESET pPrivateSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX;
|
---|
1313 | do
|
---|
1314 | {
|
---|
1315 | fRedoFromStart = false;
|
---|
1316 | iCountDown = 10240;
|
---|
1317 | pChunk = pPrivateSet->apLists[GMM_CHUNK_FREE_SET_UNUSED_LIST];
|
---|
1318 | while (pChunk)
|
---|
1319 | {
|
---|
1320 | PGMMCHUNK pNext = pChunk->pFreeNext;
|
---|
1321 | Assert(pChunk->cFree == GMM_CHUNK_NUM_PAGES);
|
---|
1322 | if ( !pGMM->fBoundMemoryMode
|
---|
1323 | || pChunk->hGVM == pGVM->hSelf)
|
---|
1324 | {
|
---|
1325 | uint64_t const idGenerationOld = pPrivateSet->idGeneration;
|
---|
1326 | if (gmmR0FreeChunk(pGMM, pGVM, pChunk, true /*fRelaxedSem*/))
|
---|
1327 | {
|
---|
1328 | /* We've left the giant mutex, restart? (+1 for our unlink) */
|
---|
1329 | fRedoFromStart = pPrivateSet->idGeneration != idGenerationOld + 1;
|
---|
1330 | if (fRedoFromStart)
|
---|
1331 | break;
|
---|
1332 | uLockNanoTS = RTTimeSystemNanoTS();
|
---|
1333 | iCountDown = 10240;
|
---|
1334 | }
|
---|
1335 | }
|
---|
1336 |
|
---|
1337 | /* Advance and maybe yield the lock. */
|
---|
1338 | pChunk = pNext;
|
---|
1339 | if (--iCountDown == 0)
|
---|
1340 | {
|
---|
1341 | uint64_t const idGenerationOld = pPrivateSet->idGeneration;
|
---|
1342 | fRedoFromStart = gmmR0MutexYield(pGMM, &uLockNanoTS)
|
---|
1343 | && pPrivateSet->idGeneration != idGenerationOld;
|
---|
1344 | if (fRedoFromStart)
|
---|
1345 | break;
|
---|
1346 | iCountDown = 10240;
|
---|
1347 | }
|
---|
1348 | }
|
---|
1349 | } while (fRedoFromStart);
|
---|
1350 |
|
---|
1351 | /*
|
---|
1352 | * Account for shared pages that weren't freed.
|
---|
1353 | */
|
---|
1354 | if (pGVM->gmm.s.Stats.cSharedPages)
|
---|
1355 | {
|
---|
1356 | Assert(pGMM->cSharedPages >= pGVM->gmm.s.Stats.cSharedPages);
|
---|
1357 | SUPR0Printf("GMMR0CleanupVM: hGVM=%#x left %#x shared pages behind!\n", pGVM->hSelf, pGVM->gmm.s.Stats.cSharedPages);
|
---|
1358 | pGMM->cLeftBehindSharedPages += pGVM->gmm.s.Stats.cSharedPages;
|
---|
1359 | }
|
---|
1360 |
|
---|
1361 | /*
|
---|
1362 | * Clean up balloon statistics in case the VM process crashed.
|
---|
1363 | */
|
---|
1364 | Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages);
|
---|
1365 | pGMM->cBalloonedPages -= pGVM->gmm.s.Stats.cBalloonedPages;
|
---|
1366 |
|
---|
1367 | /*
|
---|
1368 | * Update the over-commitment management statistics.
|
---|
1369 | */
|
---|
1370 | pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages
|
---|
1371 | + pGVM->gmm.s.Stats.Reserved.cFixedPages
|
---|
1372 | + pGVM->gmm.s.Stats.Reserved.cShadowPages;
|
---|
1373 | switch (pGVM->gmm.s.Stats.enmPolicy)
|
---|
1374 | {
|
---|
1375 | case GMMOCPOLICY_NO_OC:
|
---|
1376 | break;
|
---|
1377 | default:
|
---|
1378 | /** @todo Update GMM->cOverCommittedPages */
|
---|
1379 | break;
|
---|
1380 | }
|
---|
1381 | }
|
---|
1382 |
|
---|
1383 | /* zap the GVM data. */
|
---|
1384 | pGVM->gmm.s.Stats.enmPolicy = GMMOCPOLICY_INVALID;
|
---|
1385 | pGVM->gmm.s.Stats.enmPriority = GMMPRIORITY_INVALID;
|
---|
1386 | pGVM->gmm.s.Stats.fMayAllocate = false;
|
---|
1387 |
|
---|
1388 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
1389 | gmmR0MutexRelease(pGMM);
|
---|
1390 |
|
---|
1391 | /*
|
---|
1392 | * Destroy the spinlock.
|
---|
1393 | */
|
---|
1394 | RTSPINLOCK hSpinlock = NIL_RTSPINLOCK;
|
---|
1395 | ASMAtomicXchgHandle(&pGVM->gmm.s.hChunkTlbSpinLock, NIL_RTSPINLOCK, &hSpinlock);
|
---|
1396 | RTSpinlockDestroy(hSpinlock);
|
---|
1397 |
|
---|
1398 | LogFlow(("GMMR0CleanupVM: returns\n"));
|
---|
1399 | }
|
---|
1400 |
|
---|
1401 |
|
---|
1402 | /**
|
---|
1403 | * Scan one chunk for private pages belonging to the specified VM.
|
---|
1404 | *
|
---|
1405 | * @note This function may drop the giant mutex!
|
---|
1406 | *
|
---|
1407 | * @returns @c true if we've temporarily dropped the giant mutex, @c false if
|
---|
1408 | * we didn't.
|
---|
1409 | * @param pGMM Pointer to the GMM instance.
|
---|
1410 | * @param pGVM The global VM handle.
|
---|
1411 | * @param pChunk The chunk to scan.
|
---|
1412 | */
|
---|
1413 | static bool gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
|
---|
1414 | {
|
---|
1415 | Assert(!pGMM->fBoundMemoryMode || pChunk->hGVM == pGVM->hSelf);
|
---|
1416 |
|
---|
1417 | /*
|
---|
1418 | * Look for pages belonging to the VM.
|
---|
1419 | * (Perform some internal checks while we're scanning.)
|
---|
1420 | */
|
---|
1421 | #ifndef VBOX_STRICT
|
---|
1422 | if (pChunk->cFree != GMM_CHUNK_NUM_PAGES)
|
---|
1423 | #endif
|
---|
1424 | {
|
---|
1425 | unsigned cPrivate = 0;
|
---|
1426 | unsigned cShared = 0;
|
---|
1427 | unsigned cFree = 0;
|
---|
1428 |
|
---|
1429 | gmmR0UnlinkChunk(pChunk); /* avoiding cFreePages updates. */
|
---|
1430 |
|
---|
1431 | uint16_t hGVM = pGVM->hSelf;
|
---|
1432 | unsigned iPage = (GMM_CHUNK_SIZE >> GUEST_PAGE_SHIFT);
|
---|
1433 | while (iPage-- > 0)
|
---|
1434 | if (GMM_PAGE_IS_PRIVATE(&pChunk->aPages[iPage]))
|
---|
1435 | {
|
---|
1436 | if (pChunk->aPages[iPage].Private.hGVM == hGVM)
|
---|
1437 | {
|
---|
1438 | /*
|
---|
1439 | * Free the page.
|
---|
1440 | *
|
---|
1441 | * The reason for not using gmmR0FreePrivatePage here is that we
|
---|
1442 | * must *not* cause the chunk to be freed from under us - we're in
|
---|
1443 | * an AVL tree walk here.
|
---|
1444 | */
|
---|
1445 | pChunk->aPages[iPage].u = 0;
|
---|
1446 | pChunk->aPages[iPage].Free.u2State = GMM_PAGE_STATE_FREE;
|
---|
1447 | pChunk->aPages[iPage].Free.fZeroed = false;
|
---|
1448 | pChunk->aPages[iPage].Free.iNext = pChunk->iFreeHead;
|
---|
1449 | pChunk->iFreeHead = iPage;
|
---|
1450 | pChunk->cPrivate--;
|
---|
1451 | pChunk->cFree++;
|
---|
1452 | pGVM->gmm.s.Stats.cPrivatePages--;
|
---|
1453 | cFree++;
|
---|
1454 | }
|
---|
1455 | else
|
---|
1456 | cPrivate++;
|
---|
1457 | }
|
---|
1458 | else if (GMM_PAGE_IS_FREE(&pChunk->aPages[iPage]))
|
---|
1459 | cFree++;
|
---|
1460 | else
|
---|
1461 | cShared++;
|
---|
1462 |
|
---|
1463 | gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
|
---|
1464 |
|
---|
1465 | /*
|
---|
1466 | * Did it add up?
|
---|
1467 | */
|
---|
1468 | if (RT_UNLIKELY( pChunk->cFree != cFree
|
---|
1469 | || pChunk->cPrivate != cPrivate
|
---|
1470 | || pChunk->cShared != cShared))
|
---|
1471 | {
|
---|
1472 | SUPR0Printf("gmmR0CleanupVMScanChunk: Chunk %RKv/%#x has bogus stats - free=%d/%d private=%d/%d shared=%d/%d\n",
|
---|
1473 | pChunk, pChunk->Core.Key, pChunk->cFree, cFree, pChunk->cPrivate, cPrivate, pChunk->cShared, cShared);
|
---|
1474 | pChunk->cFree = cFree;
|
---|
1475 | pChunk->cPrivate = cPrivate;
|
---|
1476 | pChunk->cShared = cShared;
|
---|
1477 | }
|
---|
1478 | }
|
---|
1479 |
|
---|
1480 | /*
|
---|
1481 | * If not in bound memory mode, we should reset the hGVM field
|
---|
1482 | * if it has our handle in it.
|
---|
1483 | */
|
---|
1484 | if (pChunk->hGVM == pGVM->hSelf)
|
---|
1485 | {
|
---|
1486 | if (!g_pGMM->fBoundMemoryMode)
|
---|
1487 | pChunk->hGVM = NIL_GVM_HANDLE;
|
---|
1488 | else if (pChunk->cFree != GMM_CHUNK_NUM_PAGES)
|
---|
1489 | {
|
---|
1490 | SUPR0Printf("gmmR0CleanupVMScanChunk: %RKv/%#x: cFree=%#x - it should be 0 in bound mode!\n",
|
---|
1491 | pChunk, pChunk->Core.Key, pChunk->cFree);
|
---|
1492 | AssertMsgFailed(("%p/%#x: cFree=%#x - it should be 0 in bound mode!\n", pChunk, pChunk->Core.Key, pChunk->cFree));
|
---|
1493 |
|
---|
1494 | gmmR0UnlinkChunk(pChunk);
|
---|
1495 | pChunk->cFree = GMM_CHUNK_NUM_PAGES;
|
---|
1496 | gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
|
---|
1497 | }
|
---|
1498 | }
|
---|
1499 |
|
---|
1500 | /*
|
---|
1501 | * Look for a mapping belonging to the terminating VM.
|
---|
1502 | */
|
---|
1503 | GMMR0CHUNKMTXSTATE MtxState;
|
---|
1504 | gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
|
---|
1505 | unsigned cMappings = pChunk->cMappingsX;
|
---|
1506 | for (unsigned i = 0; i < cMappings; i++)
|
---|
1507 | if (pChunk->paMappingsX[i].pGVM == pGVM)
|
---|
1508 | {
|
---|
1509 | gmmR0ChunkMutexDropGiant(&MtxState);
|
---|
1510 |
|
---|
1511 | RTR0MEMOBJ hMemObj = pChunk->paMappingsX[i].hMapObj;
|
---|
1512 |
|
---|
1513 | cMappings--;
|
---|
1514 | if (i < cMappings)
|
---|
1515 | pChunk->paMappingsX[i] = pChunk->paMappingsX[cMappings];
|
---|
1516 | pChunk->paMappingsX[cMappings].pGVM = NULL;
|
---|
1517 | pChunk->paMappingsX[cMappings].hMapObj = NIL_RTR0MEMOBJ;
|
---|
1518 | Assert(pChunk->cMappingsX - 1U == cMappings);
|
---|
1519 | pChunk->cMappingsX = cMappings;
|
---|
1520 |
|
---|
1521 | int rc = RTR0MemObjFree(hMemObj, false /* fFreeMappings (NA) */);
|
---|
1522 | if (RT_FAILURE(rc))
|
---|
1523 | {
|
---|
1524 | SUPR0Printf("gmmR0CleanupVMScanChunk: %RKv/%#x: mapping #%x: RTRMemObjFree(%RKv,false) -> %d \n",
|
---|
1525 | pChunk, pChunk->Core.Key, i, hMemObj, rc);
|
---|
1526 | AssertRC(rc);
|
---|
1527 | }
|
---|
1528 |
|
---|
1529 | gmmR0ChunkMutexRelease(&MtxState, pChunk);
|
---|
1530 | return true;
|
---|
1531 | }
|
---|
1532 |
|
---|
1533 | gmmR0ChunkMutexRelease(&MtxState, pChunk);
|
---|
1534 | return false;
|
---|
1535 | }
|
---|
1536 |
|
---|
1537 |
|
---|
1538 | /**
|
---|
1539 | * The initial resource reservations.
|
---|
1540 | *
|
---|
1541 | * This will make memory reservations according to policy and priority. If there aren't
|
---|
1542 | * sufficient resources available to sustain the VM this function will fail and all
|
---|
1543 | * future allocations requests will fail as well.
|
---|
1544 | *
|
---|
1545 | * These are just the initial reservations made very very early during the VM creation
|
---|
1546 | * process and will be adjusted later in the GMMR0UpdateReservation call after the
|
---|
1547 | * ring-3 init has completed.
|
---|
1548 | *
|
---|
1549 | * @returns VBox status code.
|
---|
1550 | * @retval VERR_GMM_MEMORY_RESERVATION_DECLINED
|
---|
1551 | * @retval VERR_GMM_
|
---|
1552 | *
|
---|
1553 | * @param pGVM The global (ring-0) VM structure.
|
---|
1554 | * @param idCpu The VCPU id - must be zero.
|
---|
1555 | * @param cBasePages The number of pages that may be allocated for the base RAM and ROMs.
|
---|
1556 | * This does not include MMIO2 and similar.
|
---|
1557 | * @param cShadowPages The number of pages that may be allocated for shadow paging structures.
|
---|
1558 | * @param cFixedPages The number of pages that may be allocated for fixed objects like the
|
---|
1559 | * hyper heap, MMIO2 and similar.
|
---|
1560 | * @param enmPolicy The OC policy to use on this VM.
|
---|
1561 | * @param enmPriority The priority in an out-of-memory situation.
|
---|
1562 | *
|
---|
1563 | * @thread The creator thread / EMT(0).
|
---|
1564 | */
|
---|
1565 | GMMR0DECL(int) GMMR0InitialReservation(PGVM pGVM, VMCPUID idCpu, uint64_t cBasePages, uint32_t cShadowPages,
|
---|
1566 | uint32_t cFixedPages, GMMOCPOLICY enmPolicy, GMMPRIORITY enmPriority)
|
---|
1567 | {
|
---|
1568 | LogFlow(("GMMR0InitialReservation: pGVM=%p cBasePages=%#llx cShadowPages=%#x cFixedPages=%#x enmPolicy=%d enmPriority=%d\n",
|
---|
1569 | pGVM, cBasePages, cShadowPages, cFixedPages, enmPolicy, enmPriority));
|
---|
1570 |
|
---|
1571 | /*
|
---|
1572 | * Validate, get basics and take the semaphore.
|
---|
1573 | */
|
---|
1574 | AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID);
|
---|
1575 | PGMM pGMM;
|
---|
1576 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
1577 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
1578 | if (RT_FAILURE(rc))
|
---|
1579 | return rc;
|
---|
1580 |
|
---|
1581 | AssertReturn(cBasePages, VERR_INVALID_PARAMETER);
|
---|
1582 | AssertReturn(cShadowPages, VERR_INVALID_PARAMETER);
|
---|
1583 | AssertReturn(cFixedPages, VERR_INVALID_PARAMETER);
|
---|
1584 | AssertReturn(enmPolicy > GMMOCPOLICY_INVALID && enmPolicy < GMMOCPOLICY_END, VERR_INVALID_PARAMETER);
|
---|
1585 | AssertReturn(enmPriority > GMMPRIORITY_INVALID && enmPriority < GMMPRIORITY_END, VERR_INVALID_PARAMETER);
|
---|
1586 |
|
---|
1587 | gmmR0MutexAcquire(pGMM);
|
---|
1588 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
1589 | {
|
---|
1590 | if ( !pGVM->gmm.s.Stats.Reserved.cBasePages
|
---|
1591 | && !pGVM->gmm.s.Stats.Reserved.cFixedPages
|
---|
1592 | && !pGVM->gmm.s.Stats.Reserved.cShadowPages)
|
---|
1593 | {
|
---|
1594 | /*
|
---|
1595 | * Check if we can accommodate this.
|
---|
1596 | */
|
---|
1597 | /* ... later ... */
|
---|
1598 | if (RT_SUCCESS(rc))
|
---|
1599 | {
|
---|
1600 | /*
|
---|
1601 | * Update the records.
|
---|
1602 | */
|
---|
1603 | pGVM->gmm.s.Stats.Reserved.cBasePages = cBasePages;
|
---|
1604 | pGVM->gmm.s.Stats.Reserved.cFixedPages = cFixedPages;
|
---|
1605 | pGVM->gmm.s.Stats.Reserved.cShadowPages = cShadowPages;
|
---|
1606 | pGVM->gmm.s.Stats.enmPolicy = enmPolicy;
|
---|
1607 | pGVM->gmm.s.Stats.enmPriority = enmPriority;
|
---|
1608 | pGVM->gmm.s.Stats.fMayAllocate = true;
|
---|
1609 |
|
---|
1610 | pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages;
|
---|
1611 | pGMM->cRegisteredVMs++;
|
---|
1612 | }
|
---|
1613 | }
|
---|
1614 | else
|
---|
1615 | rc = VERR_WRONG_ORDER;
|
---|
1616 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
1617 | }
|
---|
1618 | else
|
---|
1619 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
1620 | gmmR0MutexRelease(pGMM);
|
---|
1621 | LogFlow(("GMMR0InitialReservation: returns %Rrc\n", rc));
|
---|
1622 | return rc;
|
---|
1623 | }
|
---|
1624 |
|
---|
1625 |
|
---|
1626 | /**
|
---|
1627 | * VMMR0 request wrapper for GMMR0InitialReservation.
|
---|
1628 | *
|
---|
1629 | * @returns see GMMR0InitialReservation.
|
---|
1630 | * @param pGVM The global (ring-0) VM structure.
|
---|
1631 | * @param idCpu The VCPU id.
|
---|
1632 | * @param pReq Pointer to the request packet.
|
---|
1633 | */
|
---|
1634 | GMMR0DECL(int) GMMR0InitialReservationReq(PGVM pGVM, VMCPUID idCpu, PGMMINITIALRESERVATIONREQ pReq)
|
---|
1635 | {
|
---|
1636 | /*
|
---|
1637 | * Validate input and pass it on.
|
---|
1638 | */
|
---|
1639 | AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
|
---|
1640 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
1641 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
|
---|
1642 |
|
---|
1643 | return GMMR0InitialReservation(pGVM, idCpu, pReq->cBasePages, pReq->cShadowPages,
|
---|
1644 | pReq->cFixedPages, pReq->enmPolicy, pReq->enmPriority);
|
---|
1645 | }
|
---|
1646 |
|
---|
1647 |
|
---|
1648 | /**
|
---|
1649 | * This updates the memory reservation with the additional MMIO2 and ROM pages.
|
---|
1650 | *
|
---|
1651 | * @returns VBox status code.
|
---|
1652 | * @retval VERR_GMM_MEMORY_RESERVATION_DECLINED
|
---|
1653 | *
|
---|
1654 | * @param pGVM The global (ring-0) VM structure.
|
---|
1655 | * @param idCpu The VCPU id.
|
---|
1656 | * @param cBasePages The number of pages that may be allocated for the base RAM and ROMs.
|
---|
1657 | * This does not include MMIO2 and similar.
|
---|
1658 | * @param cShadowPages The number of pages that may be allocated for shadow paging structures.
|
---|
1659 | * @param cFixedPages The number of pages that may be allocated for fixed objects like the
|
---|
1660 | * hyper heap, MMIO2 and similar.
|
---|
1661 | *
|
---|
1662 | * @thread EMT(idCpu)
|
---|
1663 | */
|
---|
1664 | GMMR0DECL(int) GMMR0UpdateReservation(PGVM pGVM, VMCPUID idCpu, uint64_t cBasePages,
|
---|
1665 | uint32_t cShadowPages, uint32_t cFixedPages)
|
---|
1666 | {
|
---|
1667 | LogFlow(("GMMR0UpdateReservation: pGVM=%p cBasePages=%#llx cShadowPages=%#x cFixedPages=%#x\n",
|
---|
1668 | pGVM, cBasePages, cShadowPages, cFixedPages));
|
---|
1669 |
|
---|
1670 | /*
|
---|
1671 | * Validate, get basics and take the semaphore.
|
---|
1672 | */
|
---|
1673 | PGMM pGMM;
|
---|
1674 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
1675 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
1676 | if (RT_FAILURE(rc))
|
---|
1677 | return rc;
|
---|
1678 |
|
---|
1679 | AssertReturn(cBasePages, VERR_INVALID_PARAMETER);
|
---|
1680 | AssertReturn(cShadowPages, VERR_INVALID_PARAMETER);
|
---|
1681 | AssertReturn(cFixedPages, VERR_INVALID_PARAMETER);
|
---|
1682 |
|
---|
1683 | gmmR0MutexAcquire(pGMM);
|
---|
1684 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
1685 | {
|
---|
1686 | if ( pGVM->gmm.s.Stats.Reserved.cBasePages
|
---|
1687 | && pGVM->gmm.s.Stats.Reserved.cFixedPages
|
---|
1688 | && pGVM->gmm.s.Stats.Reserved.cShadowPages)
|
---|
1689 | {
|
---|
1690 | /*
|
---|
1691 | * Check if we can accommodate this.
|
---|
1692 | */
|
---|
1693 | /* ... later ... */
|
---|
1694 | if (RT_SUCCESS(rc))
|
---|
1695 | {
|
---|
1696 | /*
|
---|
1697 | * Update the records.
|
---|
1698 | */
|
---|
1699 | pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages
|
---|
1700 | + pGVM->gmm.s.Stats.Reserved.cFixedPages
|
---|
1701 | + pGVM->gmm.s.Stats.Reserved.cShadowPages;
|
---|
1702 | pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages;
|
---|
1703 |
|
---|
1704 | pGVM->gmm.s.Stats.Reserved.cBasePages = cBasePages;
|
---|
1705 | pGVM->gmm.s.Stats.Reserved.cFixedPages = cFixedPages;
|
---|
1706 | pGVM->gmm.s.Stats.Reserved.cShadowPages = cShadowPages;
|
---|
1707 | }
|
---|
1708 | }
|
---|
1709 | else
|
---|
1710 | rc = VERR_WRONG_ORDER;
|
---|
1711 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
1712 | }
|
---|
1713 | else
|
---|
1714 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
1715 | gmmR0MutexRelease(pGMM);
|
---|
1716 | LogFlow(("GMMR0UpdateReservation: returns %Rrc\n", rc));
|
---|
1717 | return rc;
|
---|
1718 | }
|
---|
1719 |
|
---|
1720 |
|
---|
1721 | /**
|
---|
1722 | * VMMR0 request wrapper for GMMR0UpdateReservation.
|
---|
1723 | *
|
---|
1724 | * @returns see GMMR0UpdateReservation.
|
---|
1725 | * @param pGVM The global (ring-0) VM structure.
|
---|
1726 | * @param idCpu The VCPU id.
|
---|
1727 | * @param pReq Pointer to the request packet.
|
---|
1728 | */
|
---|
1729 | GMMR0DECL(int) GMMR0UpdateReservationReq(PGVM pGVM, VMCPUID idCpu, PGMMUPDATERESERVATIONREQ pReq)
|
---|
1730 | {
|
---|
1731 | /*
|
---|
1732 | * Validate input and pass it on.
|
---|
1733 | */
|
---|
1734 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
1735 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
|
---|
1736 |
|
---|
1737 | return GMMR0UpdateReservation(pGVM, idCpu, pReq->cBasePages, pReq->cShadowPages, pReq->cFixedPages);
|
---|
1738 | }
|
---|
1739 |
|
---|
1740 | #ifdef GMMR0_WITH_SANITY_CHECK
|
---|
1741 |
|
---|
1742 | /**
|
---|
1743 | * Performs sanity checks on a free set.
|
---|
1744 | *
|
---|
1745 | * @returns Error count.
|
---|
1746 | *
|
---|
1747 | * @param pGMM Pointer to the GMM instance.
|
---|
1748 | * @param pSet Pointer to the set.
|
---|
1749 | * @param pszSetName The set name.
|
---|
1750 | * @param pszFunction The function from which it was called.
|
---|
1751 | * @param uLine The line number.
|
---|
1752 | */
|
---|
1753 | static uint32_t gmmR0SanityCheckSet(PGMM pGMM, PGMMCHUNKFREESET pSet, const char *pszSetName,
|
---|
1754 | const char *pszFunction, unsigned uLineNo)
|
---|
1755 | {
|
---|
1756 | uint32_t cErrors = 0;
|
---|
1757 |
|
---|
1758 | /*
|
---|
1759 | * Count the free pages in all the chunks and match it against pSet->cFreePages.
|
---|
1760 | */
|
---|
1761 | uint32_t cPages = 0;
|
---|
1762 | for (unsigned i = 0; i < RT_ELEMENTS(pSet->apLists); i++)
|
---|
1763 | {
|
---|
1764 | for (PGMMCHUNK pCur = pSet->apLists[i]; pCur; pCur = pCur->pFreeNext)
|
---|
1765 | {
|
---|
1766 | /** @todo check that the chunk is hash into the right set. */
|
---|
1767 | cPages += pCur->cFree;
|
---|
1768 | }
|
---|
1769 | }
|
---|
1770 | if (RT_UNLIKELY(cPages != pSet->cFreePages))
|
---|
1771 | {
|
---|
1772 | SUPR0Printf("GMM insanity: found %#x pages in the %s set, expected %#x. (%s, line %u)\n",
|
---|
1773 | cPages, pszSetName, pSet->cFreePages, pszFunction, uLineNo);
|
---|
1774 | cErrors++;
|
---|
1775 | }
|
---|
1776 |
|
---|
1777 | return cErrors;
|
---|
1778 | }
|
---|
1779 |
|
---|
1780 |
|
---|
1781 | /**
|
---|
1782 | * Performs some sanity checks on the GMM while owning lock.
|
---|
1783 | *
|
---|
1784 | * @returns Error count.
|
---|
1785 | *
|
---|
1786 | * @param pGMM Pointer to the GMM instance.
|
---|
1787 | * @param pszFunction The function from which it is called.
|
---|
1788 | * @param uLineNo The line number.
|
---|
1789 | */
|
---|
1790 | static uint32_t gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo)
|
---|
1791 | {
|
---|
1792 | uint32_t cErrors = 0;
|
---|
1793 |
|
---|
1794 | cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->PrivateX, "private", pszFunction, uLineNo);
|
---|
1795 | cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->Shared, "shared", pszFunction, uLineNo);
|
---|
1796 | /** @todo add more sanity checks. */
|
---|
1797 |
|
---|
1798 | return cErrors;
|
---|
1799 | }
|
---|
1800 |
|
---|
1801 | #endif /* GMMR0_WITH_SANITY_CHECK */
|
---|
1802 |
|
---|
1803 | /**
|
---|
1804 | * Looks up a chunk in the tree and fill in the TLB entry for it.
|
---|
1805 | *
|
---|
1806 | * This is not expected to fail and will bitch if it does.
|
---|
1807 | *
|
---|
1808 | * @returns Pointer to the allocation chunk, NULL if not found.
|
---|
1809 | * @param pGMM Pointer to the GMM instance.
|
---|
1810 | * @param idChunk The ID of the chunk to find.
|
---|
1811 | * @param pTlbe Pointer to the TLB entry.
|
---|
1812 | *
|
---|
1813 | * @note Caller owns spinlock.
|
---|
1814 | */
|
---|
1815 | static PGMMCHUNK gmmR0GetChunkSlow(PGMM pGMM, uint32_t idChunk, PGMMCHUNKTLBE pTlbe)
|
---|
1816 | {
|
---|
1817 | PGMMCHUNK pChunk = (PGMMCHUNK)RTAvlU32Get(&pGMM->pChunks, idChunk);
|
---|
1818 | AssertMsgReturn(pChunk, ("Chunk %#x not found!\n", idChunk), NULL);
|
---|
1819 | pTlbe->idChunk = idChunk;
|
---|
1820 | pTlbe->pChunk = pChunk;
|
---|
1821 | return pChunk;
|
---|
1822 | }
|
---|
1823 |
|
---|
1824 |
|
---|
1825 | /**
|
---|
1826 | * Finds a allocation chunk, spin-locked.
|
---|
1827 | *
|
---|
1828 | * This is not expected to fail and will bitch if it does.
|
---|
1829 | *
|
---|
1830 | * @returns Pointer to the allocation chunk, NULL if not found.
|
---|
1831 | * @param pGMM Pointer to the GMM instance.
|
---|
1832 | * @param idChunk The ID of the chunk to find.
|
---|
1833 | */
|
---|
1834 | DECLINLINE(PGMMCHUNK) gmmR0GetChunkLocked(PGMM pGMM, uint32_t idChunk)
|
---|
1835 | {
|
---|
1836 | /*
|
---|
1837 | * Do a TLB lookup, branch if not in the TLB.
|
---|
1838 | */
|
---|
1839 | PGMMCHUNKTLBE pTlbe = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(idChunk)];
|
---|
1840 | PGMMCHUNK pChunk = pTlbe->pChunk;
|
---|
1841 | if ( pChunk == NULL
|
---|
1842 | || pTlbe->idChunk != idChunk)
|
---|
1843 | pChunk = gmmR0GetChunkSlow(pGMM, idChunk, pTlbe);
|
---|
1844 | return pChunk;
|
---|
1845 | }
|
---|
1846 |
|
---|
1847 |
|
---|
1848 | /**
|
---|
1849 | * Finds a allocation chunk.
|
---|
1850 | *
|
---|
1851 | * This is not expected to fail and will bitch if it does.
|
---|
1852 | *
|
---|
1853 | * @returns Pointer to the allocation chunk, NULL if not found.
|
---|
1854 | * @param pGMM Pointer to the GMM instance.
|
---|
1855 | * @param idChunk The ID of the chunk to find.
|
---|
1856 | */
|
---|
1857 | DECLINLINE(PGMMCHUNK) gmmR0GetChunk(PGMM pGMM, uint32_t idChunk)
|
---|
1858 | {
|
---|
1859 | RTSpinlockAcquire(pGMM->hSpinLockTree);
|
---|
1860 | PGMMCHUNK pChunk = gmmR0GetChunkLocked(pGMM, idChunk);
|
---|
1861 | RTSpinlockRelease(pGMM->hSpinLockTree);
|
---|
1862 | return pChunk;
|
---|
1863 | }
|
---|
1864 |
|
---|
1865 |
|
---|
1866 | /**
|
---|
1867 | * Finds a page.
|
---|
1868 | *
|
---|
1869 | * This is not expected to fail and will bitch if it does.
|
---|
1870 | *
|
---|
1871 | * @returns Pointer to the page, NULL if not found.
|
---|
1872 | * @param pGMM Pointer to the GMM instance.
|
---|
1873 | * @param idPage The ID of the page to find.
|
---|
1874 | */
|
---|
1875 | DECLINLINE(PGMMPAGE) gmmR0GetPage(PGMM pGMM, uint32_t idPage)
|
---|
1876 | {
|
---|
1877 | PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
|
---|
1878 | if (RT_LIKELY(pChunk))
|
---|
1879 | return &pChunk->aPages[idPage & GMM_PAGEID_IDX_MASK];
|
---|
1880 | return NULL;
|
---|
1881 | }
|
---|
1882 |
|
---|
1883 |
|
---|
1884 | #if 0 /* unused */
|
---|
1885 | /**
|
---|
1886 | * Gets the host physical address for a page given by it's ID.
|
---|
1887 | *
|
---|
1888 | * @returns The host physical address or NIL_RTHCPHYS.
|
---|
1889 | * @param pGMM Pointer to the GMM instance.
|
---|
1890 | * @param idPage The ID of the page to find.
|
---|
1891 | */
|
---|
1892 | DECLINLINE(RTHCPHYS) gmmR0GetPageHCPhys(PGMM pGMM, uint32_t idPage)
|
---|
1893 | {
|
---|
1894 | PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
|
---|
1895 | if (RT_LIKELY(pChunk))
|
---|
1896 | return RTR0MemObjGetPagePhysAddr(pChunk->hMemObj, idPage & GMM_PAGEID_IDX_MASK);
|
---|
1897 | return NIL_RTHCPHYS;
|
---|
1898 | }
|
---|
1899 | #endif /* unused */
|
---|
1900 |
|
---|
1901 |
|
---|
1902 | /**
|
---|
1903 | * Selects the appropriate free list given the number of free pages.
|
---|
1904 | *
|
---|
1905 | * @returns Free list index.
|
---|
1906 | * @param cFree The number of free pages in the chunk.
|
---|
1907 | */
|
---|
1908 | DECLINLINE(unsigned) gmmR0SelectFreeSetList(unsigned cFree)
|
---|
1909 | {
|
---|
1910 | unsigned iList = cFree >> GMM_CHUNK_FREE_SET_SHIFT;
|
---|
1911 | AssertMsg(iList < RT_SIZEOFMEMB(GMMCHUNKFREESET, apLists) / RT_SIZEOFMEMB(GMMCHUNKFREESET, apLists[0]),
|
---|
1912 | ("%d (%u)\n", iList, cFree));
|
---|
1913 | return iList;
|
---|
1914 | }
|
---|
1915 |
|
---|
1916 |
|
---|
1917 | /**
|
---|
1918 | * Unlinks the chunk from the free list it's currently on (if any).
|
---|
1919 | *
|
---|
1920 | * @param pChunk The allocation chunk.
|
---|
1921 | */
|
---|
1922 | DECLINLINE(void) gmmR0UnlinkChunk(PGMMCHUNK pChunk)
|
---|
1923 | {
|
---|
1924 | PGMMCHUNKFREESET pSet = pChunk->pSet;
|
---|
1925 | if (RT_LIKELY(pSet))
|
---|
1926 | {
|
---|
1927 | pSet->cFreePages -= pChunk->cFree;
|
---|
1928 | pSet->idGeneration++;
|
---|
1929 |
|
---|
1930 | PGMMCHUNK pPrev = pChunk->pFreePrev;
|
---|
1931 | PGMMCHUNK pNext = pChunk->pFreeNext;
|
---|
1932 | if (pPrev)
|
---|
1933 | pPrev->pFreeNext = pNext;
|
---|
1934 | else
|
---|
1935 | pSet->apLists[gmmR0SelectFreeSetList(pChunk->cFree)] = pNext;
|
---|
1936 | if (pNext)
|
---|
1937 | pNext->pFreePrev = pPrev;
|
---|
1938 |
|
---|
1939 | pChunk->pSet = NULL;
|
---|
1940 | pChunk->pFreeNext = NULL;
|
---|
1941 | pChunk->pFreePrev = NULL;
|
---|
1942 | }
|
---|
1943 | else
|
---|
1944 | {
|
---|
1945 | Assert(!pChunk->pFreeNext);
|
---|
1946 | Assert(!pChunk->pFreePrev);
|
---|
1947 | Assert(!pChunk->cFree);
|
---|
1948 | }
|
---|
1949 | }
|
---|
1950 |
|
---|
1951 |
|
---|
1952 | /**
|
---|
1953 | * Links the chunk onto the appropriate free list in the specified free set.
|
---|
1954 | *
|
---|
1955 | * If no free entries, it's not linked into any list.
|
---|
1956 | *
|
---|
1957 | * @param pChunk The allocation chunk.
|
---|
1958 | * @param pSet The free set.
|
---|
1959 | */
|
---|
1960 | DECLINLINE(void) gmmR0LinkChunk(PGMMCHUNK pChunk, PGMMCHUNKFREESET pSet)
|
---|
1961 | {
|
---|
1962 | Assert(!pChunk->pSet);
|
---|
1963 | Assert(!pChunk->pFreeNext);
|
---|
1964 | Assert(!pChunk->pFreePrev);
|
---|
1965 |
|
---|
1966 | if (pChunk->cFree > 0)
|
---|
1967 | {
|
---|
1968 | pChunk->pSet = pSet;
|
---|
1969 | pChunk->pFreePrev = NULL;
|
---|
1970 | unsigned const iList = gmmR0SelectFreeSetList(pChunk->cFree);
|
---|
1971 | pChunk->pFreeNext = pSet->apLists[iList];
|
---|
1972 | if (pChunk->pFreeNext)
|
---|
1973 | pChunk->pFreeNext->pFreePrev = pChunk;
|
---|
1974 | pSet->apLists[iList] = pChunk;
|
---|
1975 |
|
---|
1976 | pSet->cFreePages += pChunk->cFree;
|
---|
1977 | pSet->idGeneration++;
|
---|
1978 | }
|
---|
1979 | }
|
---|
1980 |
|
---|
1981 |
|
---|
1982 | /**
|
---|
1983 | * Links the chunk onto the appropriate free list in the specified free set.
|
---|
1984 | *
|
---|
1985 | * If no free entries, it's not linked into any list.
|
---|
1986 | *
|
---|
1987 | * @param pGMM Pointer to the GMM instance.
|
---|
1988 | * @param pGVM Pointer to the kernel-only VM instace data.
|
---|
1989 | * @param pChunk The allocation chunk.
|
---|
1990 | */
|
---|
1991 | DECLINLINE(void) gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
|
---|
1992 | {
|
---|
1993 | PGMMCHUNKFREESET pSet;
|
---|
1994 | if (pGMM->fBoundMemoryMode)
|
---|
1995 | pSet = &pGVM->gmm.s.Private;
|
---|
1996 | else if (pChunk->cShared)
|
---|
1997 | pSet = &pGMM->Shared;
|
---|
1998 | else
|
---|
1999 | pSet = &pGMM->PrivateX;
|
---|
2000 | gmmR0LinkChunk(pChunk, pSet);
|
---|
2001 | }
|
---|
2002 |
|
---|
2003 |
|
---|
2004 | /**
|
---|
2005 | * Frees a Chunk ID.
|
---|
2006 | *
|
---|
2007 | * @param pGMM Pointer to the GMM instance.
|
---|
2008 | * @param idChunk The Chunk ID to free.
|
---|
2009 | */
|
---|
2010 | static void gmmR0FreeChunkId(PGMM pGMM, uint32_t idChunk)
|
---|
2011 | {
|
---|
2012 | AssertReturnVoid(idChunk != NIL_GMM_CHUNKID);
|
---|
2013 | RTSpinlockAcquire(pGMM->hSpinLockChunkId); /* We could probably skip the locking here, I think. */
|
---|
2014 |
|
---|
2015 | AssertMsg(ASMBitTest(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk));
|
---|
2016 | ASMAtomicBitClear(&pGMM->bmChunkId[0], idChunk);
|
---|
2017 |
|
---|
2018 | RTSpinlockRelease(pGMM->hSpinLockChunkId);
|
---|
2019 | }
|
---|
2020 |
|
---|
2021 |
|
---|
2022 | /**
|
---|
2023 | * Allocates a new Chunk ID.
|
---|
2024 | *
|
---|
2025 | * @returns The Chunk ID.
|
---|
2026 | * @param pGMM Pointer to the GMM instance.
|
---|
2027 | */
|
---|
2028 | static uint32_t gmmR0AllocateChunkId(PGMM pGMM)
|
---|
2029 | {
|
---|
2030 | AssertCompile(!((GMM_CHUNKID_LAST + 1) & 31)); /* must be a multiple of 32 */
|
---|
2031 | AssertCompile(NIL_GMM_CHUNKID == 0);
|
---|
2032 |
|
---|
2033 | RTSpinlockAcquire(pGMM->hSpinLockChunkId);
|
---|
2034 |
|
---|
2035 | /*
|
---|
2036 | * Try the next sequential one.
|
---|
2037 | */
|
---|
2038 | int32_t idChunk = ++pGMM->idChunkPrev;
|
---|
2039 | if ( (uint32_t)idChunk <= GMM_CHUNKID_LAST
|
---|
2040 | && idChunk > NIL_GMM_CHUNKID)
|
---|
2041 | {
|
---|
2042 | if (!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk))
|
---|
2043 | {
|
---|
2044 | RTSpinlockRelease(pGMM->hSpinLockChunkId);
|
---|
2045 | return idChunk;
|
---|
2046 | }
|
---|
2047 |
|
---|
2048 | /*
|
---|
2049 | * Scan sequentially from the last one.
|
---|
2050 | */
|
---|
2051 | if ((uint32_t)idChunk < GMM_CHUNKID_LAST)
|
---|
2052 | {
|
---|
2053 | idChunk = ASMBitNextClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1, idChunk);
|
---|
2054 | if ( idChunk > NIL_GMM_CHUNKID
|
---|
2055 | && (uint32_t)idChunk <= GMM_CHUNKID_LAST)
|
---|
2056 | {
|
---|
2057 | AssertMsgReturnStmt(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk),
|
---|
2058 | RTSpinlockRelease(pGMM->hSpinLockChunkId), NIL_GMM_CHUNKID);
|
---|
2059 |
|
---|
2060 | pGMM->idChunkPrev = idChunk;
|
---|
2061 | RTSpinlockRelease(pGMM->hSpinLockChunkId);
|
---|
2062 | return idChunk;
|
---|
2063 | }
|
---|
2064 | }
|
---|
2065 | }
|
---|
2066 |
|
---|
2067 | /*
|
---|
2068 | * Ok, scan from the start.
|
---|
2069 | * We're not racing anyone, so there is no need to expect failures or have restart loops.
|
---|
2070 | */
|
---|
2071 | idChunk = ASMBitFirstClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1);
|
---|
2072 | AssertMsgReturnStmt(idChunk > NIL_GMM_CHUNKID && (uint32_t)idChunk <= GMM_CHUNKID_LAST, ("%#x\n", idChunk),
|
---|
2073 | RTSpinlockRelease(pGMM->hSpinLockChunkId), NIL_GVM_HANDLE);
|
---|
2074 | AssertMsgReturnStmt(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk),
|
---|
2075 | RTSpinlockRelease(pGMM->hSpinLockChunkId), NIL_GMM_CHUNKID);
|
---|
2076 |
|
---|
2077 | pGMM->idChunkPrev = idChunk;
|
---|
2078 | RTSpinlockRelease(pGMM->hSpinLockChunkId);
|
---|
2079 | return idChunk;
|
---|
2080 | }
|
---|
2081 |
|
---|
2082 |
|
---|
2083 | /**
|
---|
2084 | * Allocates one private page.
|
---|
2085 | *
|
---|
2086 | * Worker for gmmR0AllocatePages.
|
---|
2087 | *
|
---|
2088 | * @param pChunk The chunk to allocate it from.
|
---|
2089 | * @param hGVM The GVM handle of the VM requesting memory.
|
---|
2090 | * @param pPageDesc The page descriptor.
|
---|
2091 | */
|
---|
2092 | static void gmmR0AllocatePage(PGMMCHUNK pChunk, uint32_t hGVM, PGMMPAGEDESC pPageDesc)
|
---|
2093 | {
|
---|
2094 | /* update the chunk stats. */
|
---|
2095 | if (pChunk->hGVM == NIL_GVM_HANDLE)
|
---|
2096 | pChunk->hGVM = hGVM;
|
---|
2097 | Assert(pChunk->cFree);
|
---|
2098 | pChunk->cFree--;
|
---|
2099 | pChunk->cPrivate++;
|
---|
2100 |
|
---|
2101 | /* unlink the first free page. */
|
---|
2102 | const uint32_t iPage = pChunk->iFreeHead;
|
---|
2103 | AssertReleaseMsg(iPage < RT_ELEMENTS(pChunk->aPages), ("%d\n", iPage));
|
---|
2104 | PGMMPAGE pPage = &pChunk->aPages[iPage];
|
---|
2105 | Assert(GMM_PAGE_IS_FREE(pPage));
|
---|
2106 | pChunk->iFreeHead = pPage->Free.iNext;
|
---|
2107 | Log3(("A pPage=%p iPage=%#x/%#x u2State=%d iFreeHead=%#x iNext=%#x\n",
|
---|
2108 | pPage, iPage, (pChunk->Core.Key << GMM_CHUNKID_SHIFT) | iPage,
|
---|
2109 | pPage->Common.u2State, pChunk->iFreeHead, pPage->Free.iNext));
|
---|
2110 |
|
---|
2111 | bool const fZeroed = pPage->Free.fZeroed;
|
---|
2112 |
|
---|
2113 | /* make the page private. */
|
---|
2114 | pPage->u = 0;
|
---|
2115 | AssertCompile(GMM_PAGE_STATE_PRIVATE == 0);
|
---|
2116 | pPage->Private.hGVM = hGVM;
|
---|
2117 | AssertCompile(NIL_RTHCPHYS >= GMM_GCPHYS_LAST);
|
---|
2118 | AssertCompile(GMM_GCPHYS_UNSHAREABLE >= GMM_GCPHYS_LAST);
|
---|
2119 | if (pPageDesc->HCPhysGCPhys <= GMM_GCPHYS_LAST)
|
---|
2120 | pPage->Private.pfn = pPageDesc->HCPhysGCPhys >> GUEST_PAGE_SHIFT;
|
---|
2121 | else
|
---|
2122 | pPage->Private.pfn = GMM_PAGE_PFN_UNSHAREABLE; /* unshareable / unassigned - same thing. */
|
---|
2123 |
|
---|
2124 | /* update the page descriptor. */
|
---|
2125 | pPageDesc->idSharedPage = NIL_GMM_PAGEID;
|
---|
2126 | pPageDesc->idPage = (pChunk->Core.Key << GMM_CHUNKID_SHIFT) | iPage;
|
---|
2127 | RTHCPHYS const HCPhys = RTR0MemObjGetPagePhysAddr(pChunk->hMemObj, iPage);
|
---|
2128 | Assert(HCPhys != NIL_RTHCPHYS); Assert(HCPhys < NIL_GMMPAGEDESC_PHYS);
|
---|
2129 | pPageDesc->HCPhysGCPhys = HCPhys;
|
---|
2130 | pPageDesc->fZeroed = fZeroed;
|
---|
2131 | }
|
---|
2132 |
|
---|
2133 |
|
---|
2134 | /**
|
---|
2135 | * Picks the free pages from a chunk.
|
---|
2136 | *
|
---|
2137 | * @returns The new page descriptor table index.
|
---|
2138 | * @param pChunk The chunk.
|
---|
2139 | * @param hGVM The affinity of the chunk. NIL_GVM_HANDLE for no
|
---|
2140 | * affinity.
|
---|
2141 | * @param iPage The current page descriptor table index.
|
---|
2142 | * @param cPages The total number of pages to allocate.
|
---|
2143 | * @param paPages The page descriptor table (input + ouput).
|
---|
2144 | */
|
---|
2145 | static uint32_t gmmR0AllocatePagesFromChunk(PGMMCHUNK pChunk, uint16_t const hGVM, uint32_t iPage, uint32_t cPages,
|
---|
2146 | PGMMPAGEDESC paPages)
|
---|
2147 | {
|
---|
2148 | PGMMCHUNKFREESET pSet = pChunk->pSet; Assert(pSet);
|
---|
2149 | gmmR0UnlinkChunk(pChunk);
|
---|
2150 |
|
---|
2151 | for (; pChunk->cFree && iPage < cPages; iPage++)
|
---|
2152 | gmmR0AllocatePage(pChunk, hGVM, &paPages[iPage]);
|
---|
2153 |
|
---|
2154 | gmmR0LinkChunk(pChunk, pSet);
|
---|
2155 | return iPage;
|
---|
2156 | }
|
---|
2157 |
|
---|
2158 |
|
---|
2159 | /**
|
---|
2160 | * Registers a new chunk of memory.
|
---|
2161 | *
|
---|
2162 | * This is called by gmmR0AllocateOneChunk and GMMR0AllocateLargePage.
|
---|
2163 | *
|
---|
2164 | * In the GMMR0AllocateLargePage case the GMM_CHUNK_FLAGS_LARGE_PAGE flag is
|
---|
2165 | * set and the chunk will be registered as fully allocated to save time.
|
---|
2166 | *
|
---|
2167 | * @returns VBox status code. On success, the giant GMM lock will be held, the
|
---|
2168 | * caller must release it (ugly).
|
---|
2169 | * @param pGMM Pointer to the GMM instance.
|
---|
2170 | * @param pSet Pointer to the set.
|
---|
2171 | * @param hMemObj The memory object for the chunk.
|
---|
2172 | * @param hGVM The affinity of the chunk. NIL_GVM_HANDLE for no
|
---|
2173 | * affinity.
|
---|
2174 | * @param pSession Same as @a hGVM.
|
---|
2175 | * @param fChunkFlags The chunk flags, GMM_CHUNK_FLAGS_XXX.
|
---|
2176 | * @param cPages The number of pages requested. Zero for large pages.
|
---|
2177 | * @param paPages The page descriptor table (input + output). NULL for
|
---|
2178 | * large pages.
|
---|
2179 | * @param piPage The pointer to the page descriptor table index variable.
|
---|
2180 | * This will be updated. NULL for large pages.
|
---|
2181 | * @param ppChunk Chunk address (out).
|
---|
2182 | *
|
---|
2183 | * @remarks The caller must not own the giant GMM mutex.
|
---|
2184 | * The giant GMM mutex will be acquired and returned acquired in
|
---|
2185 | * the success path. On failure, no locks will be held.
|
---|
2186 | */
|
---|
2187 | static int gmmR0RegisterChunk(PGMM pGMM, PGMMCHUNKFREESET pSet, RTR0MEMOBJ hMemObj, uint16_t hGVM, PSUPDRVSESSION pSession,
|
---|
2188 | uint16_t fChunkFlags, uint32_t cPages, PGMMPAGEDESC paPages, uint32_t *piPage, PGMMCHUNK *ppChunk)
|
---|
2189 | {
|
---|
2190 | /*
|
---|
2191 | * Validate input & state.
|
---|
2192 | */
|
---|
2193 | Assert(pGMM->hMtxOwner != RTThreadNativeSelf());
|
---|
2194 | Assert(hGVM != NIL_GVM_HANDLE || pGMM->fBoundMemoryMode);
|
---|
2195 | Assert(fChunkFlags == 0 || fChunkFlags == GMM_CHUNK_FLAGS_LARGE_PAGE);
|
---|
2196 | if (!(fChunkFlags &= GMM_CHUNK_FLAGS_LARGE_PAGE))
|
---|
2197 | {
|
---|
2198 | AssertPtr(paPages);
|
---|
2199 | AssertPtr(piPage);
|
---|
2200 | Assert(cPages > 0);
|
---|
2201 | Assert(cPages > *piPage);
|
---|
2202 | }
|
---|
2203 | else
|
---|
2204 | {
|
---|
2205 | Assert(cPages == 0);
|
---|
2206 | Assert(!paPages);
|
---|
2207 | Assert(!piPage);
|
---|
2208 | }
|
---|
2209 |
|
---|
2210 | #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
2211 | /*
|
---|
2212 | * Get a ring-0 mapping of the object.
|
---|
2213 | */
|
---|
2214 | uint8_t *pbMapping = (uint8_t *)RTR0MemObjAddress(hMemObj);
|
---|
2215 | if (!pbMapping)
|
---|
2216 | {
|
---|
2217 | RTR0MEMOBJ hMapObj;
|
---|
2218 | int rc = RTR0MemObjMapKernel(&hMapObj, hMemObj, (void *)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
|
---|
2219 | if (RT_SUCCESS(rc))
|
---|
2220 | pbMapping = (uint8_t *)RTR0MemObjAddress(hMapObj);
|
---|
2221 | else
|
---|
2222 | return rc;
|
---|
2223 | AssertPtr(pbMapping);
|
---|
2224 | }
|
---|
2225 | #endif
|
---|
2226 |
|
---|
2227 | /*
|
---|
2228 | * Allocate a chunk and an ID for it.
|
---|
2229 | */
|
---|
2230 | int rc;
|
---|
2231 | PGMMCHUNK pChunk = (PGMMCHUNK)RTMemAllocZ(sizeof(*pChunk));
|
---|
2232 | if (pChunk)
|
---|
2233 | {
|
---|
2234 | pChunk->Core.Key = gmmR0AllocateChunkId(pGMM);
|
---|
2235 | if ( pChunk->Core.Key != NIL_GMM_CHUNKID
|
---|
2236 | && pChunk->Core.Key <= GMM_CHUNKID_LAST)
|
---|
2237 | {
|
---|
2238 | /*
|
---|
2239 | * Initialize it.
|
---|
2240 | */
|
---|
2241 | pChunk->hMemObj = hMemObj;
|
---|
2242 | #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
2243 | pChunk->pbMapping = pbMapping;
|
---|
2244 | #endif
|
---|
2245 | pChunk->hGVM = hGVM;
|
---|
2246 | pChunk->idNumaNode = gmmR0GetCurrentNumaNodeId();
|
---|
2247 | pChunk->iChunkMtx = UINT8_MAX;
|
---|
2248 | pChunk->fFlags = fChunkFlags;
|
---|
2249 | pChunk->uidOwner = pSession ? SUPR0GetSessionUid(pSession) : NIL_RTUID;
|
---|
2250 | /*pChunk->cShared = 0; */
|
---|
2251 |
|
---|
2252 | uint32_t const iDstPageFirst = piPage ? *piPage : cPages;
|
---|
2253 | if (!(fChunkFlags & GMM_CHUNK_FLAGS_LARGE_PAGE))
|
---|
2254 | {
|
---|
2255 | /*
|
---|
2256 | * Allocate the requested number of pages from the start of the chunk,
|
---|
2257 | * queue the rest (if any) on the free list.
|
---|
2258 | */
|
---|
2259 | uint32_t const cPagesAlloc = RT_MIN(cPages - iDstPageFirst, GMM_CHUNK_NUM_PAGES);
|
---|
2260 | pChunk->cPrivate = cPagesAlloc;
|
---|
2261 | pChunk->cFree = GMM_CHUNK_NUM_PAGES - cPagesAlloc;
|
---|
2262 | pChunk->iFreeHead = GMM_CHUNK_NUM_PAGES > cPagesAlloc ? cPagesAlloc : UINT16_MAX;
|
---|
2263 |
|
---|
2264 | /* Alloc pages: */
|
---|
2265 | uint32_t const idPageChunk = pChunk->Core.Key << GMM_CHUNKID_SHIFT;
|
---|
2266 | uint32_t iDstPage = iDstPageFirst;
|
---|
2267 | uint32_t iPage;
|
---|
2268 | for (iPage = 0; iPage < cPagesAlloc; iPage++, iDstPage++)
|
---|
2269 | {
|
---|
2270 | if (paPages[iDstPage].HCPhysGCPhys <= GMM_GCPHYS_LAST)
|
---|
2271 | pChunk->aPages[iPage].Private.pfn = paPages[iDstPage].HCPhysGCPhys >> GUEST_PAGE_SHIFT;
|
---|
2272 | else
|
---|
2273 | pChunk->aPages[iPage].Private.pfn = GMM_PAGE_PFN_UNSHAREABLE; /* unshareable / unassigned - same thing. */
|
---|
2274 | pChunk->aPages[iPage].Private.hGVM = hGVM;
|
---|
2275 | pChunk->aPages[iPage].Private.u2State = GMM_PAGE_STATE_PRIVATE;
|
---|
2276 |
|
---|
2277 | paPages[iDstPage].HCPhysGCPhys = RTR0MemObjGetPagePhysAddr(hMemObj, iPage);
|
---|
2278 | paPages[iDstPage].fZeroed = true;
|
---|
2279 | paPages[iDstPage].idPage = idPageChunk | iPage;
|
---|
2280 | paPages[iDstPage].idSharedPage = NIL_GMM_PAGEID;
|
---|
2281 | }
|
---|
2282 | *piPage = iDstPage;
|
---|
2283 |
|
---|
2284 | /* Build free list: */
|
---|
2285 | if (iPage < RT_ELEMENTS(pChunk->aPages))
|
---|
2286 | {
|
---|
2287 | Assert(pChunk->iFreeHead == iPage);
|
---|
2288 | for (; iPage < RT_ELEMENTS(pChunk->aPages) - 1; iPage++)
|
---|
2289 | {
|
---|
2290 | pChunk->aPages[iPage].Free.u2State = GMM_PAGE_STATE_FREE;
|
---|
2291 | pChunk->aPages[iPage].Free.fZeroed = true;
|
---|
2292 | pChunk->aPages[iPage].Free.iNext = iPage + 1;
|
---|
2293 | }
|
---|
2294 | pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.u2State = GMM_PAGE_STATE_FREE;
|
---|
2295 | pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.fZeroed = true;
|
---|
2296 | pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.iNext = UINT16_MAX;
|
---|
2297 | }
|
---|
2298 | else
|
---|
2299 | Assert(pChunk->iFreeHead == UINT16_MAX);
|
---|
2300 | }
|
---|
2301 | else
|
---|
2302 | {
|
---|
2303 | /*
|
---|
2304 | * Large page: Mark all pages as privately allocated (watered down gmmR0AllocatePage).
|
---|
2305 | */
|
---|
2306 | pChunk->cFree = 0;
|
---|
2307 | pChunk->cPrivate = GMM_CHUNK_NUM_PAGES;
|
---|
2308 | pChunk->iFreeHead = UINT16_MAX;
|
---|
2309 |
|
---|
2310 | for (unsigned iPage = 0; iPage < RT_ELEMENTS(pChunk->aPages); iPage++)
|
---|
2311 | {
|
---|
2312 | pChunk->aPages[iPage].Private.pfn = GMM_PAGE_PFN_UNSHAREABLE;
|
---|
2313 | pChunk->aPages[iPage].Private.hGVM = hGVM;
|
---|
2314 | pChunk->aPages[iPage].Private.u2State = GMM_PAGE_STATE_PRIVATE;
|
---|
2315 | }
|
---|
2316 | }
|
---|
2317 |
|
---|
2318 | /*
|
---|
2319 | * Zero the memory if it wasn't zeroed by the host already.
|
---|
2320 | * This simplifies keeping secret kernel bits from userland and brings
|
---|
2321 | * everyone to the same level wrt allocation zeroing.
|
---|
2322 | */
|
---|
2323 | rc = VINF_SUCCESS;
|
---|
2324 | if (!RTR0MemObjWasZeroInitialized(hMemObj))
|
---|
2325 | {
|
---|
2326 | #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
2327 | if (!(fChunkFlags & GMM_CHUNK_FLAGS_LARGE_PAGE))
|
---|
2328 | {
|
---|
2329 | for (uint32_t iPage = 0; iPage < GMM_CHUNK_SIZE / HOST_PAGE_SIZE; iPage++)
|
---|
2330 | {
|
---|
2331 | void *pvPage = NULL;
|
---|
2332 | rc = SUPR0HCPhysToVirt(RTR0MemObjGetPagePhysAddr(hMemObj, iPage), &pvPage);
|
---|
2333 | AssertRCBreak(rc);
|
---|
2334 | RT_BZERO(pvPage, HOST_PAGE_SIZE);
|
---|
2335 | }
|
---|
2336 | }
|
---|
2337 | else
|
---|
2338 | {
|
---|
2339 | /* Can do the whole large page in one go. */
|
---|
2340 | void *pvPage = NULL;
|
---|
2341 | rc = SUPR0HCPhysToVirt(RTR0MemObjGetPagePhysAddr(hMemObj, 0), &pvPage);
|
---|
2342 | AssertRC(rc);
|
---|
2343 | if (RT_SUCCESS(rc))
|
---|
2344 | RT_BZERO(pvPage, GMM_CHUNK_SIZE);
|
---|
2345 | }
|
---|
2346 | #else
|
---|
2347 | RT_BZERO(pbMapping, GMM_CHUNK_SIZE);
|
---|
2348 | #endif
|
---|
2349 | }
|
---|
2350 | if (RT_SUCCESS(rc))
|
---|
2351 | {
|
---|
2352 | *ppChunk = pChunk;
|
---|
2353 |
|
---|
2354 | /*
|
---|
2355 | * Allocate a Chunk ID and insert it into the tree.
|
---|
2356 | * This has to be done behind the mutex of course.
|
---|
2357 | */
|
---|
2358 | rc = gmmR0MutexAcquire(pGMM);
|
---|
2359 | if (RT_SUCCESS(rc))
|
---|
2360 | {
|
---|
2361 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
2362 | {
|
---|
2363 | RTSpinlockAcquire(pGMM->hSpinLockTree);
|
---|
2364 | if (RTAvlU32Insert(&pGMM->pChunks, &pChunk->Core))
|
---|
2365 | {
|
---|
2366 | pGMM->cChunks++;
|
---|
2367 | RTListAppend(&pGMM->ChunkList, &pChunk->ListNode);
|
---|
2368 | RTSpinlockRelease(pGMM->hSpinLockTree);
|
---|
2369 |
|
---|
2370 | gmmR0LinkChunk(pChunk, pSet);
|
---|
2371 |
|
---|
2372 | LogFlow(("gmmR0RegisterChunk: pChunk=%p id=%#x cChunks=%d\n", pChunk, pChunk->Core.Key, pGMM->cChunks));
|
---|
2373 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
2374 | return VINF_SUCCESS;
|
---|
2375 | }
|
---|
2376 |
|
---|
2377 | /*
|
---|
2378 | * Bail out.
|
---|
2379 | */
|
---|
2380 | RTSpinlockRelease(pGMM->hSpinLockTree);
|
---|
2381 | rc = VERR_GMM_CHUNK_INSERT;
|
---|
2382 | }
|
---|
2383 | else
|
---|
2384 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
2385 | gmmR0MutexRelease(pGMM);
|
---|
2386 | }
|
---|
2387 | *ppChunk = NULL;
|
---|
2388 | }
|
---|
2389 |
|
---|
2390 | /* Undo any page allocations. */
|
---|
2391 | if (!(fChunkFlags & GMM_CHUNK_FLAGS_LARGE_PAGE))
|
---|
2392 | {
|
---|
2393 | uint32_t const cToFree = pChunk->cPrivate;
|
---|
2394 | Assert(*piPage - iDstPageFirst == cToFree);
|
---|
2395 | for (uint32_t iDstPage = iDstPageFirst, iPage = 0; iPage < cToFree; iPage++, iDstPage++)
|
---|
2396 | {
|
---|
2397 | paPages[iDstPageFirst].fZeroed = false;
|
---|
2398 | if (pChunk->aPages[iPage].Private.pfn == GMM_PAGE_PFN_UNSHAREABLE)
|
---|
2399 | paPages[iDstPageFirst].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS;
|
---|
2400 | else
|
---|
2401 | paPages[iDstPageFirst].HCPhysGCPhys = (RTHCPHYS)pChunk->aPages[iPage].Private.pfn << GUEST_PAGE_SHIFT;
|
---|
2402 | paPages[iDstPageFirst].idPage = NIL_GMM_PAGEID;
|
---|
2403 | paPages[iDstPageFirst].idSharedPage = NIL_GMM_PAGEID;
|
---|
2404 | }
|
---|
2405 | *piPage = iDstPageFirst;
|
---|
2406 | }
|
---|
2407 |
|
---|
2408 | gmmR0FreeChunkId(pGMM, pChunk->Core.Key);
|
---|
2409 | }
|
---|
2410 | else
|
---|
2411 | rc = VERR_GMM_CHUNK_INSERT;
|
---|
2412 | RTMemFree(pChunk);
|
---|
2413 | }
|
---|
2414 | else
|
---|
2415 | rc = VERR_NO_MEMORY;
|
---|
2416 | return rc;
|
---|
2417 | }
|
---|
2418 |
|
---|
2419 |
|
---|
2420 | /**
|
---|
2421 | * Allocate a new chunk, immediately pick the requested pages from it, and adds
|
---|
2422 | * what's remaining to the specified free set.
|
---|
2423 | *
|
---|
2424 | * @note This will leave the giant mutex while allocating the new chunk!
|
---|
2425 | *
|
---|
2426 | * @returns VBox status code.
|
---|
2427 | * @param pGMM Pointer to the GMM instance data.
|
---|
2428 | * @param pGVM Pointer to the kernel-only VM instace data.
|
---|
2429 | * @param pSet Pointer to the free set.
|
---|
2430 | * @param cPages The number of pages requested.
|
---|
2431 | * @param paPages The page descriptor table (input + output).
|
---|
2432 | * @param piPage The pointer to the page descriptor table index variable.
|
---|
2433 | * This will be updated.
|
---|
2434 | */
|
---|
2435 | static int gmmR0AllocateChunkNew(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet, uint32_t cPages,
|
---|
2436 | PGMMPAGEDESC paPages, uint32_t *piPage)
|
---|
2437 | {
|
---|
2438 | gmmR0MutexRelease(pGMM);
|
---|
2439 |
|
---|
2440 | RTR0MEMOBJ hMemObj;
|
---|
2441 | int rc;
|
---|
2442 | #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
2443 | if (pGMM->fHasWorkingAllocPhysNC)
|
---|
2444 | rc = RTR0MemObjAllocPhysNC(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS);
|
---|
2445 | else
|
---|
2446 | #endif
|
---|
2447 | rc = RTR0MemObjAllocPage(&hMemObj, GMM_CHUNK_SIZE, false /*fExecutable*/);
|
---|
2448 | if (RT_SUCCESS(rc))
|
---|
2449 | {
|
---|
2450 | PGMMCHUNK pIgnored;
|
---|
2451 | rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, pGVM->pSession, 0 /*fChunkFlags*/,
|
---|
2452 | cPages, paPages, piPage, &pIgnored);
|
---|
2453 | if (RT_SUCCESS(rc))
|
---|
2454 | return VINF_SUCCESS;
|
---|
2455 |
|
---|
2456 | /* bail out */
|
---|
2457 | RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
|
---|
2458 | }
|
---|
2459 |
|
---|
2460 | int rc2 = gmmR0MutexAcquire(pGMM);
|
---|
2461 | AssertRCReturn(rc2, RT_FAILURE(rc) ? rc : rc2);
|
---|
2462 | return rc;
|
---|
2463 |
|
---|
2464 | }
|
---|
2465 |
|
---|
2466 |
|
---|
2467 | /**
|
---|
2468 | * As a last restort we'll pick any page we can get.
|
---|
2469 | *
|
---|
2470 | * @returns The new page descriptor table index.
|
---|
2471 | * @param pSet The set to pick from.
|
---|
2472 | * @param pGVM Pointer to the global VM structure.
|
---|
2473 | * @param uidSelf The UID of the caller.
|
---|
2474 | * @param iPage The current page descriptor table index.
|
---|
2475 | * @param cPages The total number of pages to allocate.
|
---|
2476 | * @param paPages The page descriptor table (input + ouput).
|
---|
2477 | */
|
---|
2478 | static uint32_t gmmR0AllocatePagesIndiscriminately(PGMMCHUNKFREESET pSet, PGVM pGVM, RTUID uidSelf,
|
---|
2479 | uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
|
---|
2480 | {
|
---|
2481 | unsigned iList = RT_ELEMENTS(pSet->apLists);
|
---|
2482 | while (iList-- > 0)
|
---|
2483 | {
|
---|
2484 | PGMMCHUNK pChunk = pSet->apLists[iList];
|
---|
2485 | while (pChunk)
|
---|
2486 | {
|
---|
2487 | PGMMCHUNK pNext = pChunk->pFreeNext;
|
---|
2488 | if ( pChunk->uidOwner == uidSelf
|
---|
2489 | || ( pChunk->cMappingsX == 0
|
---|
2490 | && pChunk->cFree == (GMM_CHUNK_SIZE >> GUEST_PAGE_SHIFT)))
|
---|
2491 | {
|
---|
2492 | iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
|
---|
2493 | if (iPage >= cPages)
|
---|
2494 | return iPage;
|
---|
2495 | }
|
---|
2496 |
|
---|
2497 | pChunk = pNext;
|
---|
2498 | }
|
---|
2499 | }
|
---|
2500 | return iPage;
|
---|
2501 | }
|
---|
2502 |
|
---|
2503 |
|
---|
2504 | /**
|
---|
2505 | * Pick pages from empty chunks on the same NUMA node.
|
---|
2506 | *
|
---|
2507 | * @returns The new page descriptor table index.
|
---|
2508 | * @param pSet The set to pick from.
|
---|
2509 | * @param pGVM Pointer to the global VM structure.
|
---|
2510 | * @param uidSelf The UID of the caller.
|
---|
2511 | * @param iPage The current page descriptor table index.
|
---|
2512 | * @param cPages The total number of pages to allocate.
|
---|
2513 | * @param paPages The page descriptor table (input + ouput).
|
---|
2514 | */
|
---|
2515 | static uint32_t gmmR0AllocatePagesFromEmptyChunksOnSameNode(PGMMCHUNKFREESET pSet, PGVM pGVM, RTUID uidSelf,
|
---|
2516 | uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
|
---|
2517 | {
|
---|
2518 | PGMMCHUNK pChunk = pSet->apLists[GMM_CHUNK_FREE_SET_UNUSED_LIST];
|
---|
2519 | if (pChunk)
|
---|
2520 | {
|
---|
2521 | uint16_t const idNumaNode = gmmR0GetCurrentNumaNodeId();
|
---|
2522 | while (pChunk)
|
---|
2523 | {
|
---|
2524 | PGMMCHUNK pNext = pChunk->pFreeNext;
|
---|
2525 |
|
---|
2526 | if ( pChunk->idNumaNode == idNumaNode
|
---|
2527 | && ( pChunk->uidOwner == uidSelf
|
---|
2528 | || pChunk->cMappingsX == 0))
|
---|
2529 | {
|
---|
2530 | pChunk->hGVM = pGVM->hSelf;
|
---|
2531 | pChunk->uidOwner = uidSelf;
|
---|
2532 | iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
|
---|
2533 | if (iPage >= cPages)
|
---|
2534 | {
|
---|
2535 | pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID;
|
---|
2536 | return iPage;
|
---|
2537 | }
|
---|
2538 | }
|
---|
2539 |
|
---|
2540 | pChunk = pNext;
|
---|
2541 | }
|
---|
2542 | }
|
---|
2543 | return iPage;
|
---|
2544 | }
|
---|
2545 |
|
---|
2546 |
|
---|
2547 | /**
|
---|
2548 | * Pick pages from non-empty chunks on the same NUMA node.
|
---|
2549 | *
|
---|
2550 | * @returns The new page descriptor table index.
|
---|
2551 | * @param pSet The set to pick from.
|
---|
2552 | * @param pGVM Pointer to the global VM structure.
|
---|
2553 | * @param uidSelf The UID of the caller.
|
---|
2554 | * @param iPage The current page descriptor table index.
|
---|
2555 | * @param cPages The total number of pages to allocate.
|
---|
2556 | * @param paPages The page descriptor table (input + ouput).
|
---|
2557 | */
|
---|
2558 | static uint32_t gmmR0AllocatePagesFromSameNode(PGMMCHUNKFREESET pSet, PGVM pGVM, RTUID const uidSelf,
|
---|
2559 | uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
|
---|
2560 | {
|
---|
2561 | /** @todo start by picking from chunks with about the right size first? */
|
---|
2562 | uint16_t const idNumaNode = gmmR0GetCurrentNumaNodeId();
|
---|
2563 | unsigned iList = GMM_CHUNK_FREE_SET_UNUSED_LIST;
|
---|
2564 | while (iList-- > 0)
|
---|
2565 | {
|
---|
2566 | PGMMCHUNK pChunk = pSet->apLists[iList];
|
---|
2567 | while (pChunk)
|
---|
2568 | {
|
---|
2569 | PGMMCHUNK pNext = pChunk->pFreeNext;
|
---|
2570 |
|
---|
2571 | if ( pChunk->idNumaNode == idNumaNode
|
---|
2572 | && pChunk->uidOwner == uidSelf)
|
---|
2573 | {
|
---|
2574 | iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
|
---|
2575 | if (iPage >= cPages)
|
---|
2576 | {
|
---|
2577 | pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID;
|
---|
2578 | return iPage;
|
---|
2579 | }
|
---|
2580 | }
|
---|
2581 |
|
---|
2582 | pChunk = pNext;
|
---|
2583 | }
|
---|
2584 | }
|
---|
2585 | return iPage;
|
---|
2586 | }
|
---|
2587 |
|
---|
2588 |
|
---|
2589 | /**
|
---|
2590 | * Pick pages that are in chunks already associated with the VM.
|
---|
2591 | *
|
---|
2592 | * @returns The new page descriptor table index.
|
---|
2593 | * @param pGMM Pointer to the GMM instance data.
|
---|
2594 | * @param pGVM Pointer to the global VM structure.
|
---|
2595 | * @param pSet The set to pick from.
|
---|
2596 | * @param iPage The current page descriptor table index.
|
---|
2597 | * @param cPages The total number of pages to allocate.
|
---|
2598 | * @param paPages The page descriptor table (input + ouput).
|
---|
2599 | */
|
---|
2600 | static uint32_t gmmR0AllocatePagesAssociatedWithVM(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet,
|
---|
2601 | uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
|
---|
2602 | {
|
---|
2603 | uint16_t const hGVM = pGVM->hSelf;
|
---|
2604 |
|
---|
2605 | /* Hint. */
|
---|
2606 | if (pGVM->gmm.s.idLastChunkHint != NIL_GMM_CHUNKID)
|
---|
2607 | {
|
---|
2608 | PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pGVM->gmm.s.idLastChunkHint);
|
---|
2609 | if (pChunk && pChunk->cFree)
|
---|
2610 | {
|
---|
2611 | iPage = gmmR0AllocatePagesFromChunk(pChunk, hGVM, iPage, cPages, paPages);
|
---|
2612 | if (iPage >= cPages)
|
---|
2613 | return iPage;
|
---|
2614 | }
|
---|
2615 | }
|
---|
2616 |
|
---|
2617 | /* Scan. */
|
---|
2618 | for (unsigned iList = 0; iList < RT_ELEMENTS(pSet->apLists); iList++)
|
---|
2619 | {
|
---|
2620 | PGMMCHUNK pChunk = pSet->apLists[iList];
|
---|
2621 | while (pChunk)
|
---|
2622 | {
|
---|
2623 | PGMMCHUNK pNext = pChunk->pFreeNext;
|
---|
2624 |
|
---|
2625 | if (pChunk->hGVM == hGVM)
|
---|
2626 | {
|
---|
2627 | iPage = gmmR0AllocatePagesFromChunk(pChunk, hGVM, iPage, cPages, paPages);
|
---|
2628 | if (iPage >= cPages)
|
---|
2629 | {
|
---|
2630 | pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID;
|
---|
2631 | return iPage;
|
---|
2632 | }
|
---|
2633 | }
|
---|
2634 |
|
---|
2635 | pChunk = pNext;
|
---|
2636 | }
|
---|
2637 | }
|
---|
2638 | return iPage;
|
---|
2639 | }
|
---|
2640 |
|
---|
2641 |
|
---|
2642 |
|
---|
2643 | /**
|
---|
2644 | * Pick pages in bound memory mode.
|
---|
2645 | *
|
---|
2646 | * @returns The new page descriptor table index.
|
---|
2647 | * @param pGVM Pointer to the global VM structure.
|
---|
2648 | * @param iPage The current page descriptor table index.
|
---|
2649 | * @param cPages The total number of pages to allocate.
|
---|
2650 | * @param paPages The page descriptor table (input + ouput).
|
---|
2651 | */
|
---|
2652 | static uint32_t gmmR0AllocatePagesInBoundMode(PGVM pGVM, uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
|
---|
2653 | {
|
---|
2654 | for (unsigned iList = 0; iList < RT_ELEMENTS(pGVM->gmm.s.Private.apLists); iList++)
|
---|
2655 | {
|
---|
2656 | PGMMCHUNK pChunk = pGVM->gmm.s.Private.apLists[iList];
|
---|
2657 | while (pChunk)
|
---|
2658 | {
|
---|
2659 | Assert(pChunk->hGVM == pGVM->hSelf);
|
---|
2660 | PGMMCHUNK pNext = pChunk->pFreeNext;
|
---|
2661 | iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
|
---|
2662 | if (iPage >= cPages)
|
---|
2663 | return iPage;
|
---|
2664 | pChunk = pNext;
|
---|
2665 | }
|
---|
2666 | }
|
---|
2667 | return iPage;
|
---|
2668 | }
|
---|
2669 |
|
---|
2670 |
|
---|
2671 | /**
|
---|
2672 | * Checks if we should start picking pages from chunks of other VMs because
|
---|
2673 | * we're getting close to the system memory or reserved limit.
|
---|
2674 | *
|
---|
2675 | * @returns @c true if we should, @c false if we should first try allocate more
|
---|
2676 | * chunks.
|
---|
2677 | */
|
---|
2678 | static bool gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLimits(PGVM pGVM)
|
---|
2679 | {
|
---|
2680 | /*
|
---|
2681 | * Don't allocate a new chunk if we're
|
---|
2682 | */
|
---|
2683 | uint64_t cPgReserved = pGVM->gmm.s.Stats.Reserved.cBasePages
|
---|
2684 | + pGVM->gmm.s.Stats.Reserved.cFixedPages
|
---|
2685 | - pGVM->gmm.s.Stats.cBalloonedPages
|
---|
2686 | /** @todo what about shared pages? */;
|
---|
2687 | uint64_t cPgAllocated = pGVM->gmm.s.Stats.Allocated.cBasePages
|
---|
2688 | + pGVM->gmm.s.Stats.Allocated.cFixedPages;
|
---|
2689 | uint64_t cPgDelta = cPgReserved - cPgAllocated;
|
---|
2690 | if (cPgDelta < GMM_CHUNK_NUM_PAGES * 4)
|
---|
2691 | return true;
|
---|
2692 | /** @todo make the threshold configurable, also test the code to see if
|
---|
2693 | * this ever kicks in (we might be reserving too much or smth). */
|
---|
2694 |
|
---|
2695 | /*
|
---|
2696 | * Check how close we're to the max memory limit and how many fragments
|
---|
2697 | * there are?...
|
---|
2698 | */
|
---|
2699 | /** @todo */
|
---|
2700 |
|
---|
2701 | return false;
|
---|
2702 | }
|
---|
2703 |
|
---|
2704 |
|
---|
2705 | /**
|
---|
2706 | * Checks if we should start picking pages from chunks of other VMs because
|
---|
2707 | * there is a lot of free pages around.
|
---|
2708 | *
|
---|
2709 | * @returns @c true if we should, @c false if we should first try allocate more
|
---|
2710 | * chunks.
|
---|
2711 | */
|
---|
2712 | static bool gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(PGMM pGMM)
|
---|
2713 | {
|
---|
2714 | /*
|
---|
2715 | * Setting the limit at 16 chunks (32 MB) at the moment.
|
---|
2716 | */
|
---|
2717 | if (pGMM->PrivateX.cFreePages >= GMM_CHUNK_NUM_PAGES * 16)
|
---|
2718 | return true;
|
---|
2719 | return false;
|
---|
2720 | }
|
---|
2721 |
|
---|
2722 |
|
---|
2723 | /**
|
---|
2724 | * Common worker for GMMR0AllocateHandyPages and GMMR0AllocatePages.
|
---|
2725 | *
|
---|
2726 | * @returns VBox status code:
|
---|
2727 | * @retval VINF_SUCCESS on success.
|
---|
2728 | * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
|
---|
2729 | * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
|
---|
2730 | * that is we're trying to allocate more than we've reserved.
|
---|
2731 | *
|
---|
2732 | * @param pGMM Pointer to the GMM instance data.
|
---|
2733 | * @param pGVM Pointer to the VM.
|
---|
2734 | * @param cPages The number of pages to allocate.
|
---|
2735 | * @param paPages Pointer to the page descriptors. See GMMPAGEDESC for
|
---|
2736 | * details on what is expected on input.
|
---|
2737 | * @param enmAccount The account to charge.
|
---|
2738 | *
|
---|
2739 | * @remarks Caller owns the giant GMM lock.
|
---|
2740 | */
|
---|
2741 | static int gmmR0AllocatePagesNew(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMPAGEDESC paPages, GMMACCOUNT enmAccount)
|
---|
2742 | {
|
---|
2743 | Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
|
---|
2744 |
|
---|
2745 | /*
|
---|
2746 | * Check allocation limits.
|
---|
2747 | */
|
---|
2748 | if (RT_LIKELY(pGMM->cAllocatedPages + cPages <= pGMM->cMaxPages))
|
---|
2749 | { /* likely */ }
|
---|
2750 | else
|
---|
2751 | return VERR_GMM_HIT_GLOBAL_LIMIT;
|
---|
2752 |
|
---|
2753 | switch (enmAccount)
|
---|
2754 | {
|
---|
2755 | case GMMACCOUNT_BASE:
|
---|
2756 | if (RT_LIKELY( pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + cPages
|
---|
2757 | <= pGVM->gmm.s.Stats.Reserved.cBasePages))
|
---|
2758 | { /* likely */ }
|
---|
2759 | else
|
---|
2760 | {
|
---|
2761 | Log(("gmmR0AllocatePages:Base: Reserved=%#llx Allocated+Ballooned+Requested=%#llx+%#llx+%#x!\n",
|
---|
2762 | pGVM->gmm.s.Stats.Reserved.cBasePages, pGVM->gmm.s.Stats.Allocated.cBasePages,
|
---|
2763 | pGVM->gmm.s.Stats.cBalloonedPages, cPages));
|
---|
2764 | return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
|
---|
2765 | }
|
---|
2766 | break;
|
---|
2767 | case GMMACCOUNT_SHADOW:
|
---|
2768 | if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cShadowPages + cPages <= pGVM->gmm.s.Stats.Reserved.cShadowPages))
|
---|
2769 | { /* likely */ }
|
---|
2770 | else
|
---|
2771 | {
|
---|
2772 | Log(("gmmR0AllocatePages:Shadow: Reserved=%#x Allocated+Requested=%#x+%#x!\n",
|
---|
2773 | pGVM->gmm.s.Stats.Reserved.cShadowPages, pGVM->gmm.s.Stats.Allocated.cShadowPages, cPages));
|
---|
2774 | return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
|
---|
2775 | }
|
---|
2776 | break;
|
---|
2777 | case GMMACCOUNT_FIXED:
|
---|
2778 | if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cFixedPages + cPages <= pGVM->gmm.s.Stats.Reserved.cFixedPages))
|
---|
2779 | { /* likely */ }
|
---|
2780 | else
|
---|
2781 | {
|
---|
2782 | Log(("gmmR0AllocatePages:Fixed: Reserved=%#x Allocated+Requested=%#x+%#x!\n",
|
---|
2783 | pGVM->gmm.s.Stats.Reserved.cFixedPages, pGVM->gmm.s.Stats.Allocated.cFixedPages, cPages));
|
---|
2784 | return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
|
---|
2785 | }
|
---|
2786 | break;
|
---|
2787 | default:
|
---|
2788 | AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
|
---|
2789 | }
|
---|
2790 |
|
---|
2791 | /*
|
---|
2792 | * Update the accounts before we proceed because we might be leaving the
|
---|
2793 | * protection of the global mutex and thus run the risk of permitting
|
---|
2794 | * too much memory to be allocated.
|
---|
2795 | */
|
---|
2796 | switch (enmAccount)
|
---|
2797 | {
|
---|
2798 | case GMMACCOUNT_BASE: pGVM->gmm.s.Stats.Allocated.cBasePages += cPages; break;
|
---|
2799 | case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages += cPages; break;
|
---|
2800 | case GMMACCOUNT_FIXED: pGVM->gmm.s.Stats.Allocated.cFixedPages += cPages; break;
|
---|
2801 | default: AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
|
---|
2802 | }
|
---|
2803 | pGVM->gmm.s.Stats.cPrivatePages += cPages;
|
---|
2804 | pGMM->cAllocatedPages += cPages;
|
---|
2805 |
|
---|
2806 | /*
|
---|
2807 | * Bound mode is also relatively straightforward.
|
---|
2808 | */
|
---|
2809 | uint32_t iPage = 0;
|
---|
2810 | int rc = VINF_SUCCESS;
|
---|
2811 | if (pGMM->fBoundMemoryMode)
|
---|
2812 | {
|
---|
2813 | iPage = gmmR0AllocatePagesInBoundMode(pGVM, iPage, cPages, paPages);
|
---|
2814 | if (iPage < cPages)
|
---|
2815 | do
|
---|
2816 | rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGVM->gmm.s.Private, cPages, paPages, &iPage);
|
---|
2817 | while (iPage < cPages && RT_SUCCESS(rc));
|
---|
2818 | }
|
---|
2819 | /*
|
---|
2820 | * Shared mode is trickier as we should try archive the same locality as
|
---|
2821 | * in bound mode, but smartly make use of non-full chunks allocated by
|
---|
2822 | * other VMs if we're low on memory.
|
---|
2823 | */
|
---|
2824 | else
|
---|
2825 | {
|
---|
2826 | RTUID const uidSelf = SUPR0GetSessionUid(pGVM->pSession);
|
---|
2827 |
|
---|
2828 | /* Pick the most optimal pages first. */
|
---|
2829 | iPage = gmmR0AllocatePagesAssociatedWithVM(pGMM, pGVM, &pGMM->PrivateX, iPage, cPages, paPages);
|
---|
2830 | if (iPage < cPages)
|
---|
2831 | {
|
---|
2832 | /* Maybe we should try getting pages from chunks "belonging" to
|
---|
2833 | other VMs before allocating more chunks? */
|
---|
2834 | bool fTriedOnSameAlready = false;
|
---|
2835 | if (gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLimits(pGVM))
|
---|
2836 | {
|
---|
2837 | iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages);
|
---|
2838 | fTriedOnSameAlready = true;
|
---|
2839 | }
|
---|
2840 |
|
---|
2841 | /* Allocate memory from empty chunks. */
|
---|
2842 | if (iPage < cPages)
|
---|
2843 | iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages);
|
---|
2844 |
|
---|
2845 | /* Grab empty shared chunks. */
|
---|
2846 | if (iPage < cPages)
|
---|
2847 | iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->Shared, pGVM, uidSelf, iPage, cPages, paPages);
|
---|
2848 |
|
---|
2849 | /* If there is a lof of free pages spread around, try not waste
|
---|
2850 | system memory on more chunks. (Should trigger defragmentation.) */
|
---|
2851 | if ( !fTriedOnSameAlready
|
---|
2852 | && gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(pGMM))
|
---|
2853 | {
|
---|
2854 | iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages);
|
---|
2855 | if (iPage < cPages)
|
---|
2856 | iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages);
|
---|
2857 | }
|
---|
2858 |
|
---|
2859 | /*
|
---|
2860 | * Ok, try allocate new chunks.
|
---|
2861 | */
|
---|
2862 | if (iPage < cPages)
|
---|
2863 | {
|
---|
2864 | do
|
---|
2865 | rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGMM->PrivateX, cPages, paPages, &iPage);
|
---|
2866 | while (iPage < cPages && RT_SUCCESS(rc));
|
---|
2867 |
|
---|
2868 | #if 0 /* We cannot mix chunks with different UIDs. */
|
---|
2869 | /* If the host is out of memory, take whatever we can get. */
|
---|
2870 | if ( (rc == VERR_NO_MEMORY || rc == VERR_NO_PHYS_MEMORY)
|
---|
2871 | && pGMM->PrivateX.cFreePages + pGMM->Shared.cFreePages >= cPages - iPage)
|
---|
2872 | {
|
---|
2873 | iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
|
---|
2874 | if (iPage < cPages)
|
---|
2875 | iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->Shared, pGVM, iPage, cPages, paPages);
|
---|
2876 | AssertRelease(iPage == cPages);
|
---|
2877 | rc = VINF_SUCCESS;
|
---|
2878 | }
|
---|
2879 | #endif
|
---|
2880 | }
|
---|
2881 | }
|
---|
2882 | }
|
---|
2883 |
|
---|
2884 | /*
|
---|
2885 | * Clean up on failure. Since this is bound to be a low-memory condition
|
---|
2886 | * we will give back any empty chunks that might be hanging around.
|
---|
2887 | */
|
---|
2888 | if (RT_SUCCESS(rc))
|
---|
2889 | { /* likely */ }
|
---|
2890 | else
|
---|
2891 | {
|
---|
2892 | /* Update the statistics. */
|
---|
2893 | pGVM->gmm.s.Stats.cPrivatePages -= cPages;
|
---|
2894 | pGMM->cAllocatedPages -= cPages - iPage;
|
---|
2895 | switch (enmAccount)
|
---|
2896 | {
|
---|
2897 | case GMMACCOUNT_BASE: pGVM->gmm.s.Stats.Allocated.cBasePages -= cPages; break;
|
---|
2898 | case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages -= cPages; break;
|
---|
2899 | case GMMACCOUNT_FIXED: pGVM->gmm.s.Stats.Allocated.cFixedPages -= cPages; break;
|
---|
2900 | default: AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
|
---|
2901 | }
|
---|
2902 |
|
---|
2903 | /* Release the pages. */
|
---|
2904 | while (iPage-- > 0)
|
---|
2905 | {
|
---|
2906 | uint32_t idPage = paPages[iPage].idPage;
|
---|
2907 | PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
|
---|
2908 | if (RT_LIKELY(pPage))
|
---|
2909 | {
|
---|
2910 | Assert(GMM_PAGE_IS_PRIVATE(pPage));
|
---|
2911 | Assert(pPage->Private.hGVM == pGVM->hSelf);
|
---|
2912 | gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage);
|
---|
2913 | }
|
---|
2914 | else
|
---|
2915 | AssertMsgFailed(("idPage=%#x\n", idPage));
|
---|
2916 |
|
---|
2917 | paPages[iPage].idPage = NIL_GMM_PAGEID;
|
---|
2918 | paPages[iPage].idSharedPage = NIL_GMM_PAGEID;
|
---|
2919 | paPages[iPage].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS;
|
---|
2920 | paPages[iPage].fZeroed = false;
|
---|
2921 | }
|
---|
2922 |
|
---|
2923 | /* Free empty chunks. */
|
---|
2924 | /** @todo */
|
---|
2925 |
|
---|
2926 | /* return the fail status on failure */
|
---|
2927 | return rc;
|
---|
2928 | }
|
---|
2929 | return VINF_SUCCESS;
|
---|
2930 | }
|
---|
2931 |
|
---|
2932 |
|
---|
2933 | /**
|
---|
2934 | * Updates the previous allocations and allocates more pages.
|
---|
2935 | *
|
---|
2936 | * The handy pages are always taken from the 'base' memory account.
|
---|
2937 | * The allocated pages are not cleared and will contains random garbage.
|
---|
2938 | *
|
---|
2939 | * @returns VBox status code:
|
---|
2940 | * @retval VINF_SUCCESS on success.
|
---|
2941 | * @retval VERR_NOT_OWNER if the caller is not an EMT.
|
---|
2942 | * @retval VERR_GMM_PAGE_NOT_FOUND if one of the pages to update wasn't found.
|
---|
2943 | * @retval VERR_GMM_PAGE_NOT_PRIVATE if one of the pages to update wasn't a
|
---|
2944 | * private page.
|
---|
2945 | * @retval VERR_GMM_PAGE_NOT_SHARED if one of the pages to update wasn't a
|
---|
2946 | * shared page.
|
---|
2947 | * @retval VERR_GMM_NOT_PAGE_OWNER if one of the pages to be updated wasn't
|
---|
2948 | * owned by the VM.
|
---|
2949 | * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
|
---|
2950 | * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
|
---|
2951 | * that is we're trying to allocate more than we've reserved.
|
---|
2952 | *
|
---|
2953 | * @param pGVM The global (ring-0) VM structure.
|
---|
2954 | * @param idCpu The VCPU id.
|
---|
2955 | * @param cPagesToUpdate The number of pages to update (starting from the head).
|
---|
2956 | * @param cPagesToAlloc The number of pages to allocate (starting from the head).
|
---|
2957 | * @param paPages The array of page descriptors.
|
---|
2958 | * See GMMPAGEDESC for details on what is expected on input.
|
---|
2959 | * @thread EMT(idCpu)
|
---|
2960 | */
|
---|
2961 | GMMR0DECL(int) GMMR0AllocateHandyPages(PGVM pGVM, VMCPUID idCpu, uint32_t cPagesToUpdate,
|
---|
2962 | uint32_t cPagesToAlloc, PGMMPAGEDESC paPages)
|
---|
2963 | {
|
---|
2964 | LogFlow(("GMMR0AllocateHandyPages: pGVM=%p cPagesToUpdate=%#x cPagesToAlloc=%#x paPages=%p\n",
|
---|
2965 | pGVM, cPagesToUpdate, cPagesToAlloc, paPages));
|
---|
2966 |
|
---|
2967 | /*
|
---|
2968 | * Validate & get basics.
|
---|
2969 | * (This is a relatively busy path, so make predictions where possible.)
|
---|
2970 | */
|
---|
2971 | PGMM pGMM;
|
---|
2972 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
2973 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
2974 | if (RT_FAILURE(rc))
|
---|
2975 | return rc;
|
---|
2976 |
|
---|
2977 | AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
|
---|
2978 | AssertMsgReturn( (cPagesToUpdate && cPagesToUpdate < 1024)
|
---|
2979 | || (cPagesToAlloc && cPagesToAlloc < 1024),
|
---|
2980 | ("cPagesToUpdate=%#x cPagesToAlloc=%#x\n", cPagesToUpdate, cPagesToAlloc),
|
---|
2981 | VERR_INVALID_PARAMETER);
|
---|
2982 |
|
---|
2983 | unsigned iPage = 0;
|
---|
2984 | for (; iPage < cPagesToUpdate; iPage++)
|
---|
2985 | {
|
---|
2986 | AssertMsgReturn( ( paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST
|
---|
2987 | && !(paPages[iPage].HCPhysGCPhys & GUEST_PAGE_OFFSET_MASK))
|
---|
2988 | || paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS
|
---|
2989 | || paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE,
|
---|
2990 | ("#%#x: %RHp\n", iPage, paPages[iPage].HCPhysGCPhys),
|
---|
2991 | VERR_INVALID_PARAMETER);
|
---|
2992 | /* ignore fZeroed here */
|
---|
2993 | AssertMsgReturn( paPages[iPage].idPage <= GMM_PAGEID_LAST
|
---|
2994 | /*|| paPages[iPage].idPage == NIL_GMM_PAGEID*/,
|
---|
2995 | ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
|
---|
2996 | AssertMsgReturn( paPages[iPage].idSharedPage == NIL_GMM_PAGEID
|
---|
2997 | || paPages[iPage].idSharedPage <= GMM_PAGEID_LAST,
|
---|
2998 | ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER);
|
---|
2999 | }
|
---|
3000 |
|
---|
3001 | for (; iPage < cPagesToAlloc; iPage++)
|
---|
3002 | {
|
---|
3003 | AssertMsgReturn(paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS, ("#%#x: %RHp\n", iPage, paPages[iPage].HCPhysGCPhys), VERR_INVALID_PARAMETER);
|
---|
3004 | AssertMsgReturn(paPages[iPage].fZeroed == false, ("#%#x: %#x\n", iPage, paPages[iPage].fZeroed), VERR_INVALID_PARAMETER);
|
---|
3005 | AssertMsgReturn(paPages[iPage].idPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
|
---|
3006 | AssertMsgReturn(paPages[iPage].idSharedPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER);
|
---|
3007 | }
|
---|
3008 |
|
---|
3009 | /*
|
---|
3010 | * Take the semaphore
|
---|
3011 | */
|
---|
3012 | VMMR0EMTBLOCKCTX Ctx;
|
---|
3013 | PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
|
---|
3014 | rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, "GMMR0AllocateHandyPages", pGMM, &Ctx);
|
---|
3015 | AssertRCReturn(rc, rc);
|
---|
3016 |
|
---|
3017 | rc = gmmR0MutexAcquire(pGMM);
|
---|
3018 | if ( RT_SUCCESS(rc)
|
---|
3019 | && GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
3020 | {
|
---|
3021 | /* No allocations before the initial reservation has been made! */
|
---|
3022 | if (RT_LIKELY( pGVM->gmm.s.Stats.Reserved.cBasePages
|
---|
3023 | && pGVM->gmm.s.Stats.Reserved.cFixedPages
|
---|
3024 | && pGVM->gmm.s.Stats.Reserved.cShadowPages))
|
---|
3025 | {
|
---|
3026 | /*
|
---|
3027 | * Perform the updates.
|
---|
3028 | * Stop on the first error.
|
---|
3029 | */
|
---|
3030 | for (iPage = 0; iPage < cPagesToUpdate; iPage++)
|
---|
3031 | {
|
---|
3032 | if (paPages[iPage].idPage != NIL_GMM_PAGEID)
|
---|
3033 | {
|
---|
3034 | PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idPage);
|
---|
3035 | if (RT_LIKELY(pPage))
|
---|
3036 | {
|
---|
3037 | if (RT_LIKELY(GMM_PAGE_IS_PRIVATE(pPage)))
|
---|
3038 | {
|
---|
3039 | if (RT_LIKELY(pPage->Private.hGVM == pGVM->hSelf))
|
---|
3040 | {
|
---|
3041 | AssertCompile(NIL_RTHCPHYS > GMM_GCPHYS_LAST && GMM_GCPHYS_UNSHAREABLE > GMM_GCPHYS_LAST);
|
---|
3042 | if (RT_LIKELY(paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST))
|
---|
3043 | pPage->Private.pfn = paPages[iPage].HCPhysGCPhys >> GUEST_PAGE_SHIFT;
|
---|
3044 | else if (paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE)
|
---|
3045 | pPage->Private.pfn = GMM_PAGE_PFN_UNSHAREABLE;
|
---|
3046 | /* else: NIL_RTHCPHYS nothing */
|
---|
3047 |
|
---|
3048 | paPages[iPage].idPage = NIL_GMM_PAGEID;
|
---|
3049 | paPages[iPage].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS;
|
---|
3050 | paPages[iPage].fZeroed = false;
|
---|
3051 | }
|
---|
3052 | else
|
---|
3053 | {
|
---|
3054 | Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not owner! hGVM=%#x hSelf=%#x\n",
|
---|
3055 | iPage, paPages[iPage].idPage, pPage->Private.hGVM, pGVM->hSelf));
|
---|
3056 | rc = VERR_GMM_NOT_PAGE_OWNER;
|
---|
3057 | break;
|
---|
3058 | }
|
---|
3059 | }
|
---|
3060 | else
|
---|
3061 | {
|
---|
3062 | Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not private! %.*Rhxs (type %d)\n", iPage, paPages[iPage].idPage, sizeof(*pPage), pPage, pPage->Common.u2State));
|
---|
3063 | rc = VERR_GMM_PAGE_NOT_PRIVATE;
|
---|
3064 | break;
|
---|
3065 | }
|
---|
3066 | }
|
---|
3067 | else
|
---|
3068 | {
|
---|
3069 | Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not found! (private)\n", iPage, paPages[iPage].idPage));
|
---|
3070 | rc = VERR_GMM_PAGE_NOT_FOUND;
|
---|
3071 | break;
|
---|
3072 | }
|
---|
3073 | }
|
---|
3074 |
|
---|
3075 | if (paPages[iPage].idSharedPage == NIL_GMM_PAGEID)
|
---|
3076 | { /* likely */ }
|
---|
3077 | else
|
---|
3078 | {
|
---|
3079 | PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idSharedPage);
|
---|
3080 | if (RT_LIKELY(pPage))
|
---|
3081 | {
|
---|
3082 | if (RT_LIKELY(GMM_PAGE_IS_SHARED(pPage)))
|
---|
3083 | {
|
---|
3084 | AssertCompile(NIL_RTHCPHYS > GMM_GCPHYS_LAST && GMM_GCPHYS_UNSHAREABLE > GMM_GCPHYS_LAST);
|
---|
3085 | Assert(pPage->Shared.cRefs);
|
---|
3086 | Assert(pGVM->gmm.s.Stats.cSharedPages);
|
---|
3087 | Assert(pGVM->gmm.s.Stats.Allocated.cBasePages);
|
---|
3088 |
|
---|
3089 | Log(("GMMR0AllocateHandyPages: free shared page %x cRefs=%d\n", paPages[iPage].idSharedPage, pPage->Shared.cRefs));
|
---|
3090 | pGVM->gmm.s.Stats.cSharedPages--;
|
---|
3091 | pGVM->gmm.s.Stats.Allocated.cBasePages--;
|
---|
3092 | if (!--pPage->Shared.cRefs)
|
---|
3093 | gmmR0FreeSharedPage(pGMM, pGVM, paPages[iPage].idSharedPage, pPage);
|
---|
3094 | else
|
---|
3095 | {
|
---|
3096 | Assert(pGMM->cDuplicatePages);
|
---|
3097 | pGMM->cDuplicatePages--;
|
---|
3098 | }
|
---|
3099 |
|
---|
3100 | paPages[iPage].idSharedPage = NIL_GMM_PAGEID;
|
---|
3101 | }
|
---|
3102 | else
|
---|
3103 | {
|
---|
3104 | Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not shared!\n", iPage, paPages[iPage].idSharedPage));
|
---|
3105 | rc = VERR_GMM_PAGE_NOT_SHARED;
|
---|
3106 | break;
|
---|
3107 | }
|
---|
3108 | }
|
---|
3109 | else
|
---|
3110 | {
|
---|
3111 | Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not found! (shared)\n", iPage, paPages[iPage].idSharedPage));
|
---|
3112 | rc = VERR_GMM_PAGE_NOT_FOUND;
|
---|
3113 | break;
|
---|
3114 | }
|
---|
3115 | }
|
---|
3116 | } /* for each page to update */
|
---|
3117 |
|
---|
3118 | if (RT_SUCCESS(rc) && cPagesToAlloc > 0)
|
---|
3119 | {
|
---|
3120 | #ifdef VBOX_STRICT
|
---|
3121 | for (iPage = 0; iPage < cPagesToAlloc; iPage++)
|
---|
3122 | {
|
---|
3123 | Assert(paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS);
|
---|
3124 | Assert(paPages[iPage].fZeroed == false);
|
---|
3125 | Assert(paPages[iPage].idPage == NIL_GMM_PAGEID);
|
---|
3126 | Assert(paPages[iPage].idSharedPage == NIL_GMM_PAGEID);
|
---|
3127 | }
|
---|
3128 | #endif
|
---|
3129 |
|
---|
3130 | /*
|
---|
3131 | * Join paths with GMMR0AllocatePages for the allocation.
|
---|
3132 | * Note! gmmR0AllocateMoreChunks may leave the protection of the mutex!
|
---|
3133 | */
|
---|
3134 | rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPagesToAlloc, paPages, GMMACCOUNT_BASE);
|
---|
3135 | }
|
---|
3136 | }
|
---|
3137 | else
|
---|
3138 | rc = VERR_WRONG_ORDER;
|
---|
3139 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
3140 | gmmR0MutexRelease(pGMM);
|
---|
3141 | }
|
---|
3142 | else if (RT_SUCCESS(rc))
|
---|
3143 | {
|
---|
3144 | gmmR0MutexRelease(pGMM);
|
---|
3145 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
3146 | }
|
---|
3147 | VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
|
---|
3148 |
|
---|
3149 | LogFlow(("GMMR0AllocateHandyPages: returns %Rrc\n", rc));
|
---|
3150 | return rc;
|
---|
3151 | }
|
---|
3152 |
|
---|
3153 |
|
---|
3154 | /**
|
---|
3155 | * Allocate one or more pages.
|
---|
3156 | *
|
---|
3157 | * This is typically used for ROMs and MMIO2 (VRAM) during VM creation.
|
---|
3158 | * The allocated pages are not cleared and will contain random garbage.
|
---|
3159 | *
|
---|
3160 | * @returns VBox status code:
|
---|
3161 | * @retval VINF_SUCCESS on success.
|
---|
3162 | * @retval VERR_NOT_OWNER if the caller is not an EMT.
|
---|
3163 | * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
|
---|
3164 | * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
|
---|
3165 | * that is we're trying to allocate more than we've reserved.
|
---|
3166 | *
|
---|
3167 | * @param pGVM The global (ring-0) VM structure.
|
---|
3168 | * @param idCpu The VCPU id.
|
---|
3169 | * @param cPages The number of pages to allocate.
|
---|
3170 | * @param paPages Pointer to the page descriptors.
|
---|
3171 | * See GMMPAGEDESC for details on what is expected on
|
---|
3172 | * input.
|
---|
3173 | * @param enmAccount The account to charge.
|
---|
3174 | *
|
---|
3175 | * @thread EMT.
|
---|
3176 | */
|
---|
3177 | GMMR0DECL(int) GMMR0AllocatePages(PGVM pGVM, VMCPUID idCpu, uint32_t cPages, PGMMPAGEDESC paPages, GMMACCOUNT enmAccount)
|
---|
3178 | {
|
---|
3179 | LogFlow(("GMMR0AllocatePages: pGVM=%p cPages=%#x paPages=%p enmAccount=%d\n", pGVM, cPages, paPages, enmAccount));
|
---|
3180 |
|
---|
3181 | /*
|
---|
3182 | * Validate, get basics and take the semaphore.
|
---|
3183 | */
|
---|
3184 | PGMM pGMM;
|
---|
3185 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
3186 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
3187 | if (RT_FAILURE(rc))
|
---|
3188 | return rc;
|
---|
3189 |
|
---|
3190 | AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
|
---|
3191 | AssertMsgReturn(enmAccount > GMMACCOUNT_INVALID && enmAccount < GMMACCOUNT_END, ("%d\n", enmAccount), VERR_INVALID_PARAMETER);
|
---|
3192 | AssertMsgReturn(cPages > 0 && cPages < RT_BIT(32 - GUEST_PAGE_SHIFT), ("%#x\n", cPages), VERR_INVALID_PARAMETER);
|
---|
3193 |
|
---|
3194 | for (unsigned iPage = 0; iPage < cPages; iPage++)
|
---|
3195 | {
|
---|
3196 | AssertMsgReturn( paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS
|
---|
3197 | || paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE
|
---|
3198 | || ( enmAccount == GMMACCOUNT_BASE
|
---|
3199 | && paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST
|
---|
3200 | && !(paPages[iPage].HCPhysGCPhys & GUEST_PAGE_OFFSET_MASK)),
|
---|
3201 | ("#%#x: %RHp enmAccount=%d\n", iPage, paPages[iPage].HCPhysGCPhys, enmAccount),
|
---|
3202 | VERR_INVALID_PARAMETER);
|
---|
3203 | AssertMsgReturn(paPages[iPage].fZeroed == false, ("#%#x: %#x\n", iPage, paPages[iPage].fZeroed), VERR_INVALID_PARAMETER);
|
---|
3204 | AssertMsgReturn(paPages[iPage].idPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
|
---|
3205 | AssertMsgReturn(paPages[iPage].idSharedPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER);
|
---|
3206 | }
|
---|
3207 |
|
---|
3208 | /*
|
---|
3209 | * Grab the giant mutex and get working.
|
---|
3210 | */
|
---|
3211 | gmmR0MutexAcquire(pGMM);
|
---|
3212 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
3213 | {
|
---|
3214 |
|
---|
3215 | /* No allocations before the initial reservation has been made! */
|
---|
3216 | if (RT_LIKELY( pGVM->gmm.s.Stats.Reserved.cBasePages
|
---|
3217 | && pGVM->gmm.s.Stats.Reserved.cFixedPages
|
---|
3218 | && pGVM->gmm.s.Stats.Reserved.cShadowPages))
|
---|
3219 | rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPages, paPages, enmAccount);
|
---|
3220 | else
|
---|
3221 | rc = VERR_WRONG_ORDER;
|
---|
3222 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
3223 | }
|
---|
3224 | else
|
---|
3225 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
3226 | gmmR0MutexRelease(pGMM);
|
---|
3227 |
|
---|
3228 | LogFlow(("GMMR0AllocatePages: returns %Rrc\n", rc));
|
---|
3229 | return rc;
|
---|
3230 | }
|
---|
3231 |
|
---|
3232 |
|
---|
3233 | /**
|
---|
3234 | * VMMR0 request wrapper for GMMR0AllocatePages.
|
---|
3235 | *
|
---|
3236 | * @returns see GMMR0AllocatePages.
|
---|
3237 | * @param pGVM The global (ring-0) VM structure.
|
---|
3238 | * @param idCpu The VCPU id.
|
---|
3239 | * @param pReq Pointer to the request packet.
|
---|
3240 | */
|
---|
3241 | GMMR0DECL(int) GMMR0AllocatePagesReq(PGVM pGVM, VMCPUID idCpu, PGMMALLOCATEPAGESREQ pReq)
|
---|
3242 | {
|
---|
3243 | /*
|
---|
3244 | * Validate input and pass it on.
|
---|
3245 | */
|
---|
3246 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
3247 | AssertMsgReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF(GMMALLOCATEPAGESREQ, aPages[0]),
|
---|
3248 | ("%#x < %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF(GMMALLOCATEPAGESREQ, aPages[0])),
|
---|
3249 | VERR_INVALID_PARAMETER);
|
---|
3250 | AssertMsgReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMALLOCATEPAGESREQ, aPages[pReq->cPages]),
|
---|
3251 | ("%#x != %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF_DYN(GMMALLOCATEPAGESREQ, aPages[pReq->cPages])),
|
---|
3252 | VERR_INVALID_PARAMETER);
|
---|
3253 |
|
---|
3254 | return GMMR0AllocatePages(pGVM, idCpu, pReq->cPages, &pReq->aPages[0], pReq->enmAccount);
|
---|
3255 | }
|
---|
3256 |
|
---|
3257 |
|
---|
3258 | /**
|
---|
3259 | * Allocate a large page to represent guest RAM
|
---|
3260 | *
|
---|
3261 | * The allocated pages are zeroed upon return.
|
---|
3262 | *
|
---|
3263 | * @returns VBox status code:
|
---|
3264 | * @retval VINF_SUCCESS on success.
|
---|
3265 | * @retval VERR_NOT_OWNER if the caller is not an EMT.
|
---|
3266 | * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
|
---|
3267 | * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
|
---|
3268 | * that is we're trying to allocate more than we've reserved.
|
---|
3269 | * @retval VERR_TRY_AGAIN if the host is temporarily out of large pages.
|
---|
3270 | * @returns see GMMR0AllocatePages.
|
---|
3271 | *
|
---|
3272 | * @param pGVM The global (ring-0) VM structure.
|
---|
3273 | * @param idCpu The VCPU id.
|
---|
3274 | * @param cbPage Large page size.
|
---|
3275 | * @param pIdPage Where to return the GMM page ID of the page.
|
---|
3276 | * @param pHCPhys Where to return the host physical address of the page.
|
---|
3277 | */
|
---|
3278 | GMMR0DECL(int) GMMR0AllocateLargePage(PGVM pGVM, VMCPUID idCpu, uint32_t cbPage, uint32_t *pIdPage, RTHCPHYS *pHCPhys)
|
---|
3279 | {
|
---|
3280 | LogFlow(("GMMR0AllocateLargePage: pGVM=%p cbPage=%x\n", pGVM, cbPage));
|
---|
3281 |
|
---|
3282 | AssertPtrReturn(pIdPage, VERR_INVALID_PARAMETER);
|
---|
3283 | *pIdPage = NIL_GMM_PAGEID;
|
---|
3284 | AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
|
---|
3285 | *pHCPhys = NIL_RTHCPHYS;
|
---|
3286 | AssertReturn(cbPage == GMM_CHUNK_SIZE, VERR_INVALID_PARAMETER);
|
---|
3287 |
|
---|
3288 | /*
|
---|
3289 | * Validate GVM + idCpu, get basics and take the semaphore.
|
---|
3290 | */
|
---|
3291 | PGMM pGMM;
|
---|
3292 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
3293 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
3294 | AssertRCReturn(rc, rc);
|
---|
3295 |
|
---|
3296 | VMMR0EMTBLOCKCTX Ctx;
|
---|
3297 | PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
|
---|
3298 | rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, "GMMR0AllocateLargePage", pGMM, &Ctx);
|
---|
3299 | AssertRCReturn(rc, rc);
|
---|
3300 |
|
---|
3301 | rc = gmmR0MutexAcquire(pGMM);
|
---|
3302 | if (RT_SUCCESS(rc))
|
---|
3303 | {
|
---|
3304 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
3305 | {
|
---|
3306 | /*
|
---|
3307 | * Check the quota.
|
---|
3308 | */
|
---|
3309 | /** @todo r=bird: Quota checking could be done w/o the giant mutex but using
|
---|
3310 | * a VM specific mutex... */
|
---|
3311 | if (RT_LIKELY( pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + GMM_CHUNK_NUM_PAGES
|
---|
3312 | <= pGVM->gmm.s.Stats.Reserved.cBasePages))
|
---|
3313 | {
|
---|
3314 | /*
|
---|
3315 | * Allocate a new large page chunk.
|
---|
3316 | *
|
---|
3317 | * Note! We leave the giant GMM lock temporarily as the allocation might
|
---|
3318 | * take a long time. gmmR0RegisterChunk will retake it (ugly).
|
---|
3319 | */
|
---|
3320 | AssertCompile(GMM_CHUNK_SIZE == _2M);
|
---|
3321 | gmmR0MutexRelease(pGMM);
|
---|
3322 |
|
---|
3323 | RTR0MEMOBJ hMemObj;
|
---|
3324 | rc = RTR0MemObjAllocLarge(&hMemObj, GMM_CHUNK_SIZE, GMM_CHUNK_SIZE, RTMEMOBJ_ALLOC_LARGE_F_FAST);
|
---|
3325 | if (RT_SUCCESS(rc))
|
---|
3326 | {
|
---|
3327 | *pHCPhys = RTR0MemObjGetPagePhysAddr(hMemObj, 0);
|
---|
3328 |
|
---|
3329 | /*
|
---|
3330 | * Register the chunk as fully allocated.
|
---|
3331 | * Note! As mentioned above, this will return owning the mutex on success.
|
---|
3332 | */
|
---|
3333 | PGMMCHUNK pChunk = NULL;
|
---|
3334 | PGMMCHUNKFREESET const pSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX;
|
---|
3335 | rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, pGVM->pSession, GMM_CHUNK_FLAGS_LARGE_PAGE,
|
---|
3336 | 0 /*cPages*/, NULL /*paPages*/, NULL /*piPage*/, &pChunk);
|
---|
3337 | if (RT_SUCCESS(rc))
|
---|
3338 | {
|
---|
3339 | /*
|
---|
3340 | * The gmmR0RegisterChunk call already marked all pages allocated,
|
---|
3341 | * so we just have to fill in the return values and update stats now.
|
---|
3342 | */
|
---|
3343 | *pIdPage = pChunk->Core.Key << GMM_CHUNKID_SHIFT;
|
---|
3344 |
|
---|
3345 | /* Update accounting. */
|
---|
3346 | pGVM->gmm.s.Stats.Allocated.cBasePages += GMM_CHUNK_NUM_PAGES;
|
---|
3347 | pGVM->gmm.s.Stats.cPrivatePages += GMM_CHUNK_NUM_PAGES;
|
---|
3348 | pGMM->cAllocatedPages += GMM_CHUNK_NUM_PAGES;
|
---|
3349 |
|
---|
3350 | gmmR0LinkChunk(pChunk, pSet);
|
---|
3351 | gmmR0MutexRelease(pGMM);
|
---|
3352 |
|
---|
3353 | VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
|
---|
3354 | LogFlow(("GMMR0AllocateLargePage: returns VINF_SUCCESS\n"));
|
---|
3355 | return VINF_SUCCESS;
|
---|
3356 | }
|
---|
3357 |
|
---|
3358 | /*
|
---|
3359 | * Bail out.
|
---|
3360 | */
|
---|
3361 | RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
|
---|
3362 | *pHCPhys = NIL_RTHCPHYS;
|
---|
3363 | }
|
---|
3364 | /** @todo r=bird: Turn VERR_NO_MEMORY etc into VERR_TRY_AGAIN? Docs say we
|
---|
3365 | * return it, but I am sure IPRT doesn't... */
|
---|
3366 | }
|
---|
3367 | else
|
---|
3368 | {
|
---|
3369 | Log(("GMMR0AllocateLargePage: Reserved=%#llx Allocated+Requested=%#llx+%#x!\n",
|
---|
3370 | pGVM->gmm.s.Stats.Reserved.cBasePages, pGVM->gmm.s.Stats.Allocated.cBasePages, GMM_CHUNK_NUM_PAGES));
|
---|
3371 | gmmR0MutexRelease(pGMM);
|
---|
3372 | rc = VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
|
---|
3373 | }
|
---|
3374 | }
|
---|
3375 | else
|
---|
3376 | {
|
---|
3377 | gmmR0MutexRelease(pGMM);
|
---|
3378 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
3379 | }
|
---|
3380 | }
|
---|
3381 |
|
---|
3382 | VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
|
---|
3383 | LogFlow(("GMMR0AllocateLargePage: returns %Rrc\n", rc));
|
---|
3384 | return rc;
|
---|
3385 | }
|
---|
3386 |
|
---|
3387 |
|
---|
3388 | /**
|
---|
3389 | * Free a large page.
|
---|
3390 | *
|
---|
3391 | * @returns VBox status code:
|
---|
3392 | * @param pGVM The global (ring-0) VM structure.
|
---|
3393 | * @param idCpu The VCPU id.
|
---|
3394 | * @param idPage The large page id.
|
---|
3395 | */
|
---|
3396 | GMMR0DECL(int) GMMR0FreeLargePage(PGVM pGVM, VMCPUID idCpu, uint32_t idPage)
|
---|
3397 | {
|
---|
3398 | LogFlow(("GMMR0FreeLargePage: pGVM=%p idPage=%x\n", pGVM, idPage));
|
---|
3399 |
|
---|
3400 | /*
|
---|
3401 | * Validate, get basics and take the semaphore.
|
---|
3402 | */
|
---|
3403 | PGMM pGMM;
|
---|
3404 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
3405 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
3406 | if (RT_FAILURE(rc))
|
---|
3407 | return rc;
|
---|
3408 |
|
---|
3409 | gmmR0MutexAcquire(pGMM);
|
---|
3410 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
3411 | {
|
---|
3412 | const unsigned cPages = GMM_CHUNK_NUM_PAGES;
|
---|
3413 |
|
---|
3414 | if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages < cPages))
|
---|
3415 | {
|
---|
3416 | Log(("GMMR0FreeLargePage: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cBasePages, cPages));
|
---|
3417 | gmmR0MutexRelease(pGMM);
|
---|
3418 | return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
|
---|
3419 | }
|
---|
3420 |
|
---|
3421 | PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
|
---|
3422 | if (RT_LIKELY( pPage
|
---|
3423 | && GMM_PAGE_IS_PRIVATE(pPage)))
|
---|
3424 | {
|
---|
3425 | PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
|
---|
3426 | Assert(pChunk);
|
---|
3427 | Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
|
---|
3428 | Assert(pChunk->cPrivate > 0);
|
---|
3429 |
|
---|
3430 | /* Release the memory immediately. */
|
---|
3431 | gmmR0FreeChunk(pGMM, NULL, pChunk, false /*fRelaxedSem*/); /** @todo this can be relaxed too! */
|
---|
3432 |
|
---|
3433 | /* Update accounting. */
|
---|
3434 | pGVM->gmm.s.Stats.Allocated.cBasePages -= cPages;
|
---|
3435 | pGVM->gmm.s.Stats.cPrivatePages -= cPages;
|
---|
3436 | pGMM->cAllocatedPages -= cPages;
|
---|
3437 | }
|
---|
3438 | else
|
---|
3439 | rc = VERR_GMM_PAGE_NOT_FOUND;
|
---|
3440 | }
|
---|
3441 | else
|
---|
3442 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
3443 |
|
---|
3444 | gmmR0MutexRelease(pGMM);
|
---|
3445 | LogFlow(("GMMR0FreeLargePage: returns %Rrc\n", rc));
|
---|
3446 | return rc;
|
---|
3447 | }
|
---|
3448 |
|
---|
3449 |
|
---|
3450 | /**
|
---|
3451 | * VMMR0 request wrapper for GMMR0FreeLargePage.
|
---|
3452 | *
|
---|
3453 | * @returns see GMMR0FreeLargePage.
|
---|
3454 | * @param pGVM The global (ring-0) VM structure.
|
---|
3455 | * @param idCpu The VCPU id.
|
---|
3456 | * @param pReq Pointer to the request packet.
|
---|
3457 | */
|
---|
3458 | GMMR0DECL(int) GMMR0FreeLargePageReq(PGVM pGVM, VMCPUID idCpu, PGMMFREELARGEPAGEREQ pReq)
|
---|
3459 | {
|
---|
3460 | /*
|
---|
3461 | * Validate input and pass it on.
|
---|
3462 | */
|
---|
3463 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
3464 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMFREEPAGESREQ),
|
---|
3465 | ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(GMMFREEPAGESREQ)),
|
---|
3466 | VERR_INVALID_PARAMETER);
|
---|
3467 |
|
---|
3468 | return GMMR0FreeLargePage(pGVM, idCpu, pReq->idPage);
|
---|
3469 | }
|
---|
3470 |
|
---|
3471 |
|
---|
3472 | /**
|
---|
3473 | * @callback_method_impl{FNGVMMR0ENUMCALLBACK,
|
---|
3474 | * Used by gmmR0FreeChunkFlushPerVmTlbs().}
|
---|
3475 | */
|
---|
3476 | static DECLCALLBACK(int) gmmR0InvalidatePerVmChunkTlbCallback(PGVM pGVM, void *pvUser)
|
---|
3477 | {
|
---|
3478 | RT_NOREF(pvUser);
|
---|
3479 | if (pGVM->gmm.s.hChunkTlbSpinLock != NIL_RTSPINLOCK)
|
---|
3480 | {
|
---|
3481 | RTSpinlockAcquire(pGVM->gmm.s.hChunkTlbSpinLock);
|
---|
3482 | uintptr_t i = RT_ELEMENTS(pGVM->gmm.s.aChunkTlbEntries);
|
---|
3483 | while (i-- > 0)
|
---|
3484 | {
|
---|
3485 | pGVM->gmm.s.aChunkTlbEntries[i].idGeneration = UINT64_MAX;
|
---|
3486 | pGVM->gmm.s.aChunkTlbEntries[i].pChunk = NULL;
|
---|
3487 | }
|
---|
3488 | RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock);
|
---|
3489 | }
|
---|
3490 | return VINF_SUCCESS;
|
---|
3491 | }
|
---|
3492 |
|
---|
3493 |
|
---|
3494 | /**
|
---|
3495 | * Called by gmmR0FreeChunk when we reach the threshold for wrapping around the
|
---|
3496 | * free generation ID value.
|
---|
3497 | *
|
---|
3498 | * This is done at 2^62 - 1, which allows us to drop all locks and as it will
|
---|
3499 | * take a while before 12 exa (2 305 843 009 213 693 952) calls to
|
---|
3500 | * gmmR0FreeChunk can be made and causes a real wrap-around. We do two
|
---|
3501 | * invalidation passes and resets the generation ID between then. This will
|
---|
3502 | * make sure there are no false positives.
|
---|
3503 | *
|
---|
3504 | * @param pGMM Pointer to the GMM instance.
|
---|
3505 | */
|
---|
3506 | static void gmmR0FreeChunkFlushPerVmTlbs(PGMM pGMM)
|
---|
3507 | {
|
---|
3508 | /*
|
---|
3509 | * First invalidation pass.
|
---|
3510 | */
|
---|
3511 | int rc = GVMMR0EnumVMs(gmmR0InvalidatePerVmChunkTlbCallback, NULL);
|
---|
3512 | AssertRCSuccess(rc);
|
---|
3513 |
|
---|
3514 | /*
|
---|
3515 | * Reset the generation number.
|
---|
3516 | */
|
---|
3517 | RTSpinlockAcquire(pGMM->hSpinLockTree);
|
---|
3518 | ASMAtomicWriteU64(&pGMM->idFreeGeneration, 1);
|
---|
3519 | RTSpinlockRelease(pGMM->hSpinLockTree);
|
---|
3520 |
|
---|
3521 | /*
|
---|
3522 | * Second invalidation pass.
|
---|
3523 | */
|
---|
3524 | rc = GVMMR0EnumVMs(gmmR0InvalidatePerVmChunkTlbCallback, NULL);
|
---|
3525 | AssertRCSuccess(rc);
|
---|
3526 | }
|
---|
3527 |
|
---|
3528 |
|
---|
3529 | /**
|
---|
3530 | * Frees a chunk, giving it back to the host OS.
|
---|
3531 | *
|
---|
3532 | * @param pGMM Pointer to the GMM instance.
|
---|
3533 | * @param pGVM This is set when called from GMMR0CleanupVM so we can
|
---|
3534 | * unmap and free the chunk in one go.
|
---|
3535 | * @param pChunk The chunk to free.
|
---|
3536 | * @param fRelaxedSem Whether we can release the semaphore while doing the
|
---|
3537 | * freeing (@c true) or not.
|
---|
3538 | */
|
---|
3539 | static bool gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem)
|
---|
3540 | {
|
---|
3541 | Assert(pChunk->Core.Key != NIL_GMM_CHUNKID);
|
---|
3542 |
|
---|
3543 | GMMR0CHUNKMTXSTATE MtxState;
|
---|
3544 | gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
|
---|
3545 |
|
---|
3546 | /*
|
---|
3547 | * Cleanup hack! Unmap the chunk from the callers address space.
|
---|
3548 | * This shouldn't happen, so screw lock contention...
|
---|
3549 | */
|
---|
3550 | if (pChunk->cMappingsX && pGVM)
|
---|
3551 | gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk);
|
---|
3552 |
|
---|
3553 | /*
|
---|
3554 | * If there are current mappings of the chunk, then request the
|
---|
3555 | * VMs to unmap them. Reposition the chunk in the free list so
|
---|
3556 | * it won't be a likely candidate for allocations.
|
---|
3557 | */
|
---|
3558 | if (pChunk->cMappingsX)
|
---|
3559 | {
|
---|
3560 | /** @todo R0 -> VM request */
|
---|
3561 | /* The chunk can be mapped by more than one VM if fBoundMemoryMode is false! */
|
---|
3562 | Log(("gmmR0FreeChunk: chunk still has %d mappings; don't free!\n", pChunk->cMappingsX));
|
---|
3563 | gmmR0ChunkMutexRelease(&MtxState, pChunk);
|
---|
3564 | return false;
|
---|
3565 | }
|
---|
3566 |
|
---|
3567 |
|
---|
3568 | /*
|
---|
3569 | * Save and trash the handle.
|
---|
3570 | */
|
---|
3571 | RTR0MEMOBJ const hMemObj = pChunk->hMemObj;
|
---|
3572 | pChunk->hMemObj = NIL_RTR0MEMOBJ;
|
---|
3573 |
|
---|
3574 | /*
|
---|
3575 | * Unlink it from everywhere.
|
---|
3576 | */
|
---|
3577 | gmmR0UnlinkChunk(pChunk);
|
---|
3578 |
|
---|
3579 | RTSpinlockAcquire(pGMM->hSpinLockTree);
|
---|
3580 |
|
---|
3581 | RTListNodeRemove(&pChunk->ListNode);
|
---|
3582 |
|
---|
3583 | PAVLU32NODECORE pCore = RTAvlU32Remove(&pGMM->pChunks, pChunk->Core.Key);
|
---|
3584 | Assert(pCore == &pChunk->Core); NOREF(pCore);
|
---|
3585 |
|
---|
3586 | PGMMCHUNKTLBE pTlbe = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(pChunk->Core.Key)];
|
---|
3587 | if (pTlbe->pChunk == pChunk)
|
---|
3588 | {
|
---|
3589 | pTlbe->idChunk = NIL_GMM_CHUNKID;
|
---|
3590 | pTlbe->pChunk = NULL;
|
---|
3591 | }
|
---|
3592 |
|
---|
3593 | Assert(pGMM->cChunks > 0);
|
---|
3594 | pGMM->cChunks--;
|
---|
3595 |
|
---|
3596 | uint64_t const idFreeGeneration = ASMAtomicIncU64(&pGMM->idFreeGeneration);
|
---|
3597 |
|
---|
3598 | RTSpinlockRelease(pGMM->hSpinLockTree);
|
---|
3599 |
|
---|
3600 | pGMM->cFreedChunks++;
|
---|
3601 |
|
---|
3602 | /* Drop the lock. */
|
---|
3603 | gmmR0ChunkMutexRelease(&MtxState, NULL);
|
---|
3604 | if (fRelaxedSem)
|
---|
3605 | gmmR0MutexRelease(pGMM);
|
---|
3606 |
|
---|
3607 | /*
|
---|
3608 | * Flush per VM chunk TLBs if we're getting remotely close to a generation wraparound.
|
---|
3609 | */
|
---|
3610 | if (idFreeGeneration == UINT64_MAX / 4)
|
---|
3611 | gmmR0FreeChunkFlushPerVmTlbs(pGMM);
|
---|
3612 |
|
---|
3613 | /*
|
---|
3614 | * Free the Chunk ID and all memory associated with the chunk.
|
---|
3615 | */
|
---|
3616 | gmmR0FreeChunkId(pGMM, pChunk->Core.Key);
|
---|
3617 | pChunk->Core.Key = NIL_GMM_CHUNKID;
|
---|
3618 |
|
---|
3619 | RTMemFree(pChunk->paMappingsX);
|
---|
3620 | pChunk->paMappingsX = NULL;
|
---|
3621 |
|
---|
3622 | RTMemFree(pChunk);
|
---|
3623 |
|
---|
3624 | #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
3625 | int rc = RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
|
---|
3626 | #else
|
---|
3627 | int rc = RTR0MemObjFree(hMemObj, false /* fFreeMappings */);
|
---|
3628 | #endif
|
---|
3629 | AssertLogRelRC(rc);
|
---|
3630 |
|
---|
3631 | if (fRelaxedSem)
|
---|
3632 | gmmR0MutexAcquire(pGMM);
|
---|
3633 | return fRelaxedSem;
|
---|
3634 | }
|
---|
3635 |
|
---|
3636 |
|
---|
3637 | /**
|
---|
3638 | * Free page worker.
|
---|
3639 | *
|
---|
3640 | * The caller does all the statistic decrementing, we do all the incrementing.
|
---|
3641 | *
|
---|
3642 | * @param pGMM Pointer to the GMM instance data.
|
---|
3643 | * @param pGVM Pointer to the GVM instance.
|
---|
3644 | * @param pChunk Pointer to the chunk this page belongs to.
|
---|
3645 | * @param idPage The Page ID.
|
---|
3646 | * @param pPage Pointer to the page.
|
---|
3647 | */
|
---|
3648 | static void gmmR0FreePageWorker(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, uint32_t idPage, PGMMPAGE pPage)
|
---|
3649 | {
|
---|
3650 | Log3(("F pPage=%p iPage=%#x/%#x u2State=%d iFreeHead=%#x\n",
|
---|
3651 | pPage, pPage - &pChunk->aPages[0], idPage, pPage->Common.u2State, pChunk->iFreeHead)); NOREF(idPage);
|
---|
3652 |
|
---|
3653 | /*
|
---|
3654 | * Put the page on the free list.
|
---|
3655 | */
|
---|
3656 | pPage->u = 0;
|
---|
3657 | pPage->Free.u2State = GMM_PAGE_STATE_FREE;
|
---|
3658 | pPage->Free.fZeroed = false;
|
---|
3659 | Assert(pChunk->iFreeHead < RT_ELEMENTS(pChunk->aPages) || pChunk->iFreeHead == UINT16_MAX);
|
---|
3660 | pPage->Free.iNext = pChunk->iFreeHead;
|
---|
3661 | pChunk->iFreeHead = pPage - &pChunk->aPages[0];
|
---|
3662 |
|
---|
3663 | /*
|
---|
3664 | * Update statistics (the cShared/cPrivate stats are up to date already),
|
---|
3665 | * and relink the chunk if necessary.
|
---|
3666 | */
|
---|
3667 | unsigned const cFree = pChunk->cFree;
|
---|
3668 | if ( !cFree
|
---|
3669 | || gmmR0SelectFreeSetList(cFree) != gmmR0SelectFreeSetList(cFree + 1))
|
---|
3670 | {
|
---|
3671 | gmmR0UnlinkChunk(pChunk);
|
---|
3672 | pChunk->cFree++;
|
---|
3673 | gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
|
---|
3674 | }
|
---|
3675 | else
|
---|
3676 | {
|
---|
3677 | pChunk->cFree = cFree + 1;
|
---|
3678 | pChunk->pSet->cFreePages++;
|
---|
3679 | }
|
---|
3680 |
|
---|
3681 | /*
|
---|
3682 | * If the chunk becomes empty, consider giving memory back to the host OS.
|
---|
3683 | *
|
---|
3684 | * The current strategy is to try give it back if there are other chunks
|
---|
3685 | * in this free list, meaning if there are at least 240 free pages in this
|
---|
3686 | * category. Note that since there are probably mappings of the chunk,
|
---|
3687 | * it won't be freed up instantly, which probably screws up this logic
|
---|
3688 | * a bit...
|
---|
3689 | */
|
---|
3690 | /** @todo Do this on the way out. */
|
---|
3691 | if (RT_LIKELY( pChunk->cFree != GMM_CHUNK_NUM_PAGES
|
---|
3692 | || pChunk->pFreeNext == NULL
|
---|
3693 | || pChunk->pFreePrev == NULL /** @todo this is probably misfiring, see reset... */))
|
---|
3694 | { /* likely */ }
|
---|
3695 | else
|
---|
3696 | gmmR0FreeChunk(pGMM, NULL, pChunk, false);
|
---|
3697 | }
|
---|
3698 |
|
---|
3699 |
|
---|
3700 | /**
|
---|
3701 | * Frees a shared page, the page is known to exist and be valid and such.
|
---|
3702 | *
|
---|
3703 | * @param pGMM Pointer to the GMM instance.
|
---|
3704 | * @param pGVM Pointer to the GVM instance.
|
---|
3705 | * @param idPage The page id.
|
---|
3706 | * @param pPage The page structure.
|
---|
3707 | */
|
---|
3708 | DECLINLINE(void) gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage)
|
---|
3709 | {
|
---|
3710 | PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
|
---|
3711 | Assert(pChunk);
|
---|
3712 | Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
|
---|
3713 | Assert(pChunk->cShared > 0);
|
---|
3714 | Assert(pGMM->cSharedPages > 0);
|
---|
3715 | Assert(pGMM->cAllocatedPages > 0);
|
---|
3716 | Assert(!pPage->Shared.cRefs);
|
---|
3717 |
|
---|
3718 | pChunk->cShared--;
|
---|
3719 | pGMM->cAllocatedPages--;
|
---|
3720 | pGMM->cSharedPages--;
|
---|
3721 | gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage);
|
---|
3722 | }
|
---|
3723 |
|
---|
3724 |
|
---|
3725 | /**
|
---|
3726 | * Frees a private page, the page is known to exist and be valid and such.
|
---|
3727 | *
|
---|
3728 | * @param pGMM Pointer to the GMM instance.
|
---|
3729 | * @param pGVM Pointer to the GVM instance.
|
---|
3730 | * @param idPage The page id.
|
---|
3731 | * @param pPage The page structure.
|
---|
3732 | */
|
---|
3733 | DECLINLINE(void) gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage)
|
---|
3734 | {
|
---|
3735 | PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
|
---|
3736 | Assert(pChunk);
|
---|
3737 | Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
|
---|
3738 | Assert(pChunk->cPrivate > 0);
|
---|
3739 | Assert(pGMM->cAllocatedPages > 0);
|
---|
3740 |
|
---|
3741 | pChunk->cPrivate--;
|
---|
3742 | pGMM->cAllocatedPages--;
|
---|
3743 | gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage);
|
---|
3744 | }
|
---|
3745 |
|
---|
3746 |
|
---|
3747 | /**
|
---|
3748 | * Common worker for GMMR0FreePages and GMMR0BalloonedPages.
|
---|
3749 | *
|
---|
3750 | * @returns VBox status code:
|
---|
3751 | * @retval xxx
|
---|
3752 | *
|
---|
3753 | * @param pGMM Pointer to the GMM instance data.
|
---|
3754 | * @param pGVM Pointer to the VM.
|
---|
3755 | * @param cPages The number of pages to free.
|
---|
3756 | * @param paPages Pointer to the page descriptors.
|
---|
3757 | * @param enmAccount The account this relates to.
|
---|
3758 | */
|
---|
3759 | static int gmmR0FreePages(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMFREEPAGEDESC paPages, GMMACCOUNT enmAccount)
|
---|
3760 | {
|
---|
3761 | /*
|
---|
3762 | * Check that the request isn't impossible wrt to the account status.
|
---|
3763 | */
|
---|
3764 | switch (enmAccount)
|
---|
3765 | {
|
---|
3766 | case GMMACCOUNT_BASE:
|
---|
3767 | if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages < cPages))
|
---|
3768 | {
|
---|
3769 | Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cBasePages, cPages));
|
---|
3770 | return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
|
---|
3771 | }
|
---|
3772 | break;
|
---|
3773 | case GMMACCOUNT_SHADOW:
|
---|
3774 | if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cShadowPages < cPages))
|
---|
3775 | {
|
---|
3776 | Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cShadowPages, cPages));
|
---|
3777 | return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
|
---|
3778 | }
|
---|
3779 | break;
|
---|
3780 | case GMMACCOUNT_FIXED:
|
---|
3781 | if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cFixedPages < cPages))
|
---|
3782 | {
|
---|
3783 | Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cFixedPages, cPages));
|
---|
3784 | return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
|
---|
3785 | }
|
---|
3786 | break;
|
---|
3787 | default:
|
---|
3788 | AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
|
---|
3789 | }
|
---|
3790 |
|
---|
3791 | /*
|
---|
3792 | * Walk the descriptors and free the pages.
|
---|
3793 | *
|
---|
3794 | * Statistics (except the account) are being updated as we go along,
|
---|
3795 | * unlike the alloc code. Also, stop on the first error.
|
---|
3796 | */
|
---|
3797 | int rc = VINF_SUCCESS;
|
---|
3798 | uint32_t iPage;
|
---|
3799 | for (iPage = 0; iPage < cPages; iPage++)
|
---|
3800 | {
|
---|
3801 | uint32_t idPage = paPages[iPage].idPage;
|
---|
3802 | PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
|
---|
3803 | if (RT_LIKELY(pPage))
|
---|
3804 | {
|
---|
3805 | if (RT_LIKELY(GMM_PAGE_IS_PRIVATE(pPage)))
|
---|
3806 | {
|
---|
3807 | if (RT_LIKELY(pPage->Private.hGVM == pGVM->hSelf))
|
---|
3808 | {
|
---|
3809 | Assert(pGVM->gmm.s.Stats.cPrivatePages);
|
---|
3810 | pGVM->gmm.s.Stats.cPrivatePages--;
|
---|
3811 | gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage);
|
---|
3812 | }
|
---|
3813 | else
|
---|
3814 | {
|
---|
3815 | Log(("gmmR0AllocatePages: #%#x/%#x: not owner! hGVM=%#x hSelf=%#x\n", iPage, idPage,
|
---|
3816 | pPage->Private.hGVM, pGVM->hSelf));
|
---|
3817 | rc = VERR_GMM_NOT_PAGE_OWNER;
|
---|
3818 | break;
|
---|
3819 | }
|
---|
3820 | }
|
---|
3821 | else if (RT_LIKELY(GMM_PAGE_IS_SHARED(pPage)))
|
---|
3822 | {
|
---|
3823 | Assert(pGVM->gmm.s.Stats.cSharedPages);
|
---|
3824 | Assert(pPage->Shared.cRefs);
|
---|
3825 | #if defined(VBOX_WITH_PAGE_SHARING) && defined(VBOX_STRICT)
|
---|
3826 | if (pPage->Shared.u14Checksum)
|
---|
3827 | {
|
---|
3828 | uint32_t uChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage);
|
---|
3829 | uChecksum &= UINT32_C(0x00003fff);
|
---|
3830 | AssertMsg(!uChecksum || uChecksum == pPage->Shared.u14Checksum,
|
---|
3831 | ("%#x vs %#x - idPage=%#x\n", uChecksum, pPage->Shared.u14Checksum, idPage));
|
---|
3832 | }
|
---|
3833 | #endif
|
---|
3834 | pGVM->gmm.s.Stats.cSharedPages--;
|
---|
3835 | if (!--pPage->Shared.cRefs)
|
---|
3836 | gmmR0FreeSharedPage(pGMM, pGVM, idPage, pPage);
|
---|
3837 | else
|
---|
3838 | {
|
---|
3839 | Assert(pGMM->cDuplicatePages);
|
---|
3840 | pGMM->cDuplicatePages--;
|
---|
3841 | }
|
---|
3842 | }
|
---|
3843 | else
|
---|
3844 | {
|
---|
3845 | Log(("gmmR0AllocatePages: #%#x/%#x: already free!\n", iPage, idPage));
|
---|
3846 | rc = VERR_GMM_PAGE_ALREADY_FREE;
|
---|
3847 | break;
|
---|
3848 | }
|
---|
3849 | }
|
---|
3850 | else
|
---|
3851 | {
|
---|
3852 | Log(("gmmR0AllocatePages: #%#x/%#x: not found!\n", iPage, idPage));
|
---|
3853 | rc = VERR_GMM_PAGE_NOT_FOUND;
|
---|
3854 | break;
|
---|
3855 | }
|
---|
3856 | paPages[iPage].idPage = NIL_GMM_PAGEID;
|
---|
3857 | }
|
---|
3858 |
|
---|
3859 | /*
|
---|
3860 | * Update the account.
|
---|
3861 | */
|
---|
3862 | switch (enmAccount)
|
---|
3863 | {
|
---|
3864 | case GMMACCOUNT_BASE: pGVM->gmm.s.Stats.Allocated.cBasePages -= iPage; break;
|
---|
3865 | case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages -= iPage; break;
|
---|
3866 | case GMMACCOUNT_FIXED: pGVM->gmm.s.Stats.Allocated.cFixedPages -= iPage; break;
|
---|
3867 | default:
|
---|
3868 | AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
|
---|
3869 | }
|
---|
3870 |
|
---|
3871 | /*
|
---|
3872 | * Any threshold stuff to be done here?
|
---|
3873 | */
|
---|
3874 |
|
---|
3875 | return rc;
|
---|
3876 | }
|
---|
3877 |
|
---|
3878 |
|
---|
3879 | /**
|
---|
3880 | * Free one or more pages.
|
---|
3881 | *
|
---|
3882 | * This is typically used at reset time or power off.
|
---|
3883 | *
|
---|
3884 | * @returns VBox status code:
|
---|
3885 | * @retval xxx
|
---|
3886 | *
|
---|
3887 | * @param pGVM The global (ring-0) VM structure.
|
---|
3888 | * @param idCpu The VCPU id.
|
---|
3889 | * @param cPages The number of pages to allocate.
|
---|
3890 | * @param paPages Pointer to the page descriptors containing the page IDs
|
---|
3891 | * for each page.
|
---|
3892 | * @param enmAccount The account this relates to.
|
---|
3893 | * @thread EMT.
|
---|
3894 | */
|
---|
3895 | GMMR0DECL(int) GMMR0FreePages(PGVM pGVM, VMCPUID idCpu, uint32_t cPages, PGMMFREEPAGEDESC paPages, GMMACCOUNT enmAccount)
|
---|
3896 | {
|
---|
3897 | LogFlow(("GMMR0FreePages: pGVM=%p cPages=%#x paPages=%p enmAccount=%d\n", pGVM, cPages, paPages, enmAccount));
|
---|
3898 |
|
---|
3899 | /*
|
---|
3900 | * Validate input and get the basics.
|
---|
3901 | */
|
---|
3902 | PGMM pGMM;
|
---|
3903 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
3904 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
3905 | if (RT_FAILURE(rc))
|
---|
3906 | return rc;
|
---|
3907 |
|
---|
3908 | AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
|
---|
3909 | AssertMsgReturn(enmAccount > GMMACCOUNT_INVALID && enmAccount < GMMACCOUNT_END, ("%d\n", enmAccount), VERR_INVALID_PARAMETER);
|
---|
3910 | AssertMsgReturn(cPages > 0 && cPages < RT_BIT(32 - GUEST_PAGE_SHIFT), ("%#x\n", cPages), VERR_INVALID_PARAMETER);
|
---|
3911 |
|
---|
3912 | for (unsigned iPage = 0; iPage < cPages; iPage++)
|
---|
3913 | AssertMsgReturn( paPages[iPage].idPage <= GMM_PAGEID_LAST
|
---|
3914 | /*|| paPages[iPage].idPage == NIL_GMM_PAGEID*/,
|
---|
3915 | ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
|
---|
3916 |
|
---|
3917 | /*
|
---|
3918 | * Take the semaphore and call the worker function.
|
---|
3919 | */
|
---|
3920 | gmmR0MutexAcquire(pGMM);
|
---|
3921 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
3922 | {
|
---|
3923 | rc = gmmR0FreePages(pGMM, pGVM, cPages, paPages, enmAccount);
|
---|
3924 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
3925 | }
|
---|
3926 | else
|
---|
3927 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
3928 | gmmR0MutexRelease(pGMM);
|
---|
3929 | LogFlow(("GMMR0FreePages: returns %Rrc\n", rc));
|
---|
3930 | return rc;
|
---|
3931 | }
|
---|
3932 |
|
---|
3933 |
|
---|
3934 | /**
|
---|
3935 | * VMMR0 request wrapper for GMMR0FreePages.
|
---|
3936 | *
|
---|
3937 | * @returns see GMMR0FreePages.
|
---|
3938 | * @param pGVM The global (ring-0) VM structure.
|
---|
3939 | * @param idCpu The VCPU id.
|
---|
3940 | * @param pReq Pointer to the request packet.
|
---|
3941 | */
|
---|
3942 | GMMR0DECL(int) GMMR0FreePagesReq(PGVM pGVM, VMCPUID idCpu, PGMMFREEPAGESREQ pReq)
|
---|
3943 | {
|
---|
3944 | /*
|
---|
3945 | * Validate input and pass it on.
|
---|
3946 | */
|
---|
3947 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
3948 | AssertMsgReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF(GMMFREEPAGESREQ, aPages[0]),
|
---|
3949 | ("%#x < %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF(GMMFREEPAGESREQ, aPages[0])),
|
---|
3950 | VERR_INVALID_PARAMETER);
|
---|
3951 | AssertMsgReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[pReq->cPages]),
|
---|
3952 | ("%#x != %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[pReq->cPages])),
|
---|
3953 | VERR_INVALID_PARAMETER);
|
---|
3954 |
|
---|
3955 | return GMMR0FreePages(pGVM, idCpu, pReq->cPages, &pReq->aPages[0], pReq->enmAccount);
|
---|
3956 | }
|
---|
3957 |
|
---|
3958 |
|
---|
3959 | /**
|
---|
3960 | * Report back on a memory ballooning request.
|
---|
3961 | *
|
---|
3962 | * The request may or may not have been initiated by the GMM. If it was initiated
|
---|
3963 | * by the GMM it is important that this function is called even if no pages were
|
---|
3964 | * ballooned.
|
---|
3965 | *
|
---|
3966 | * @returns VBox status code:
|
---|
3967 | * @retval VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH
|
---|
3968 | * @retval VERR_GMM_ATTEMPT_TO_DEFLATE_TOO_MUCH
|
---|
3969 | * @retval VERR_GMM_OVERCOMMITTED_TRY_AGAIN_IN_A_BIT - reset condition
|
---|
3970 | * indicating that we won't necessarily have sufficient RAM to boot
|
---|
3971 | * the VM again and that it should pause until this changes (we'll try
|
---|
3972 | * balloon some other VM). (For standard deflate we have little choice
|
---|
3973 | * but to hope the VM won't use the memory that was returned to it.)
|
---|
3974 | *
|
---|
3975 | * @param pGVM The global (ring-0) VM structure.
|
---|
3976 | * @param idCpu The VCPU id.
|
---|
3977 | * @param enmAction Inflate/deflate/reset.
|
---|
3978 | * @param cBalloonedPages The number of pages that was ballooned.
|
---|
3979 | *
|
---|
3980 | * @thread EMT(idCpu)
|
---|
3981 | */
|
---|
3982 | GMMR0DECL(int) GMMR0BalloonedPages(PGVM pGVM, VMCPUID idCpu, GMMBALLOONACTION enmAction, uint32_t cBalloonedPages)
|
---|
3983 | {
|
---|
3984 | LogFlow(("GMMR0BalloonedPages: pGVM=%p enmAction=%d cBalloonedPages=%#x\n",
|
---|
3985 | pGVM, enmAction, cBalloonedPages));
|
---|
3986 |
|
---|
3987 | AssertMsgReturn(cBalloonedPages < RT_BIT(32 - GUEST_PAGE_SHIFT), ("%#x\n", cBalloonedPages), VERR_INVALID_PARAMETER);
|
---|
3988 |
|
---|
3989 | /*
|
---|
3990 | * Validate input and get the basics.
|
---|
3991 | */
|
---|
3992 | PGMM pGMM;
|
---|
3993 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
3994 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
3995 | if (RT_FAILURE(rc))
|
---|
3996 | return rc;
|
---|
3997 |
|
---|
3998 | /*
|
---|
3999 | * Take the semaphore and do some more validations.
|
---|
4000 | */
|
---|
4001 | gmmR0MutexAcquire(pGMM);
|
---|
4002 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
4003 | {
|
---|
4004 | switch (enmAction)
|
---|
4005 | {
|
---|
4006 | case GMMBALLOONACTION_INFLATE:
|
---|
4007 | {
|
---|
4008 | if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + cBalloonedPages
|
---|
4009 | <= pGVM->gmm.s.Stats.Reserved.cBasePages))
|
---|
4010 | {
|
---|
4011 | /*
|
---|
4012 | * Record the ballooned memory.
|
---|
4013 | */
|
---|
4014 | pGMM->cBalloonedPages += cBalloonedPages;
|
---|
4015 | if (pGVM->gmm.s.Stats.cReqBalloonedPages)
|
---|
4016 | {
|
---|
4017 | /* Codepath never taken. Might be interesting in the future to request ballooned memory from guests in low memory conditions.. */
|
---|
4018 | AssertFailed();
|
---|
4019 |
|
---|
4020 | pGVM->gmm.s.Stats.cBalloonedPages += cBalloonedPages;
|
---|
4021 | pGVM->gmm.s.Stats.cReqActuallyBalloonedPages += cBalloonedPages;
|
---|
4022 | Log(("GMMR0BalloonedPages: +%#x - Global=%#llx / VM: Total=%#llx Req=%#llx Actual=%#llx (pending)\n",
|
---|
4023 | cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages,
|
---|
4024 | pGVM->gmm.s.Stats.cReqBalloonedPages, pGVM->gmm.s.Stats.cReqActuallyBalloonedPages));
|
---|
4025 | }
|
---|
4026 | else
|
---|
4027 | {
|
---|
4028 | pGVM->gmm.s.Stats.cBalloonedPages += cBalloonedPages;
|
---|
4029 | Log(("GMMR0BalloonedPages: +%#x - Global=%#llx / VM: Total=%#llx (user)\n",
|
---|
4030 | cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages));
|
---|
4031 | }
|
---|
4032 | }
|
---|
4033 | else
|
---|
4034 | {
|
---|
4035 | Log(("GMMR0BalloonedPages: cBasePages=%#llx Total=%#llx cBalloonedPages=%#llx Reserved=%#llx\n",
|
---|
4036 | pGVM->gmm.s.Stats.Allocated.cBasePages, pGVM->gmm.s.Stats.cBalloonedPages, cBalloonedPages,
|
---|
4037 | pGVM->gmm.s.Stats.Reserved.cBasePages));
|
---|
4038 | rc = VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
|
---|
4039 | }
|
---|
4040 | break;
|
---|
4041 | }
|
---|
4042 |
|
---|
4043 | case GMMBALLOONACTION_DEFLATE:
|
---|
4044 | {
|
---|
4045 | /* Deflate. */
|
---|
4046 | if (pGVM->gmm.s.Stats.cBalloonedPages >= cBalloonedPages)
|
---|
4047 | {
|
---|
4048 | /*
|
---|
4049 | * Record the ballooned memory.
|
---|
4050 | */
|
---|
4051 | Assert(pGMM->cBalloonedPages >= cBalloonedPages);
|
---|
4052 | pGMM->cBalloonedPages -= cBalloonedPages;
|
---|
4053 | pGVM->gmm.s.Stats.cBalloonedPages -= cBalloonedPages;
|
---|
4054 | if (pGVM->gmm.s.Stats.cReqDeflatePages)
|
---|
4055 | {
|
---|
4056 | AssertFailed(); /* This is path is for later. */
|
---|
4057 | Log(("GMMR0BalloonedPages: -%#x - Global=%#llx / VM: Total=%#llx Req=%#llx\n",
|
---|
4058 | cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages, pGVM->gmm.s.Stats.cReqDeflatePages));
|
---|
4059 |
|
---|
4060 | /*
|
---|
4061 | * Anything we need to do here now when the request has been completed?
|
---|
4062 | */
|
---|
4063 | pGVM->gmm.s.Stats.cReqDeflatePages = 0;
|
---|
4064 | }
|
---|
4065 | else
|
---|
4066 | Log(("GMMR0BalloonedPages: -%#x - Global=%#llx / VM: Total=%#llx (user)\n",
|
---|
4067 | cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages));
|
---|
4068 | }
|
---|
4069 | else
|
---|
4070 | {
|
---|
4071 | Log(("GMMR0BalloonedPages: Total=%#llx cBalloonedPages=%#llx\n", pGVM->gmm.s.Stats.cBalloonedPages, cBalloonedPages));
|
---|
4072 | rc = VERR_GMM_ATTEMPT_TO_DEFLATE_TOO_MUCH;
|
---|
4073 | }
|
---|
4074 | break;
|
---|
4075 | }
|
---|
4076 |
|
---|
4077 | case GMMBALLOONACTION_RESET:
|
---|
4078 | {
|
---|
4079 | /* Reset to an empty balloon. */
|
---|
4080 | Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages);
|
---|
4081 |
|
---|
4082 | pGMM->cBalloonedPages -= pGVM->gmm.s.Stats.cBalloonedPages;
|
---|
4083 | pGVM->gmm.s.Stats.cBalloonedPages = 0;
|
---|
4084 | break;
|
---|
4085 | }
|
---|
4086 |
|
---|
4087 | default:
|
---|
4088 | rc = VERR_INVALID_PARAMETER;
|
---|
4089 | break;
|
---|
4090 | }
|
---|
4091 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
4092 | }
|
---|
4093 | else
|
---|
4094 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
4095 |
|
---|
4096 | gmmR0MutexRelease(pGMM);
|
---|
4097 | LogFlow(("GMMR0BalloonedPages: returns %Rrc\n", rc));
|
---|
4098 | return rc;
|
---|
4099 | }
|
---|
4100 |
|
---|
4101 |
|
---|
4102 | /**
|
---|
4103 | * VMMR0 request wrapper for GMMR0BalloonedPages.
|
---|
4104 | *
|
---|
4105 | * @returns see GMMR0BalloonedPages.
|
---|
4106 | * @param pGVM The global (ring-0) VM structure.
|
---|
4107 | * @param idCpu The VCPU id.
|
---|
4108 | * @param pReq Pointer to the request packet.
|
---|
4109 | */
|
---|
4110 | GMMR0DECL(int) GMMR0BalloonedPagesReq(PGVM pGVM, VMCPUID idCpu, PGMMBALLOONEDPAGESREQ pReq)
|
---|
4111 | {
|
---|
4112 | /*
|
---|
4113 | * Validate input and pass it on.
|
---|
4114 | */
|
---|
4115 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
4116 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMBALLOONEDPAGESREQ),
|
---|
4117 | ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMBALLOONEDPAGESREQ)),
|
---|
4118 | VERR_INVALID_PARAMETER);
|
---|
4119 |
|
---|
4120 | return GMMR0BalloonedPages(pGVM, idCpu, pReq->enmAction, pReq->cBalloonedPages);
|
---|
4121 | }
|
---|
4122 |
|
---|
4123 |
|
---|
4124 | /**
|
---|
4125 | * Return memory statistics for the hypervisor
|
---|
4126 | *
|
---|
4127 | * @returns VBox status code.
|
---|
4128 | * @param pReq Pointer to the request packet.
|
---|
4129 | */
|
---|
4130 | GMMR0DECL(int) GMMR0QueryHypervisorMemoryStatsReq(PGMMMEMSTATSREQ pReq)
|
---|
4131 | {
|
---|
4132 | /*
|
---|
4133 | * Validate input and pass it on.
|
---|
4134 | */
|
---|
4135 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
4136 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMMEMSTATSREQ),
|
---|
4137 | ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMMEMSTATSREQ)),
|
---|
4138 | VERR_INVALID_PARAMETER);
|
---|
4139 |
|
---|
4140 | /*
|
---|
4141 | * Validate input and get the basics.
|
---|
4142 | */
|
---|
4143 | PGMM pGMM;
|
---|
4144 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
4145 | pReq->cAllocPages = pGMM->cAllocatedPages;
|
---|
4146 | pReq->cFreePages = (pGMM->cChunks << (GMM_CHUNK_SHIFT - GUEST_PAGE_SHIFT)) - pGMM->cAllocatedPages;
|
---|
4147 | pReq->cBalloonedPages = pGMM->cBalloonedPages;
|
---|
4148 | pReq->cMaxPages = pGMM->cMaxPages;
|
---|
4149 | pReq->cSharedPages = pGMM->cDuplicatePages;
|
---|
4150 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
4151 |
|
---|
4152 | return VINF_SUCCESS;
|
---|
4153 | }
|
---|
4154 |
|
---|
4155 |
|
---|
4156 | /**
|
---|
4157 | * Return memory statistics for the VM
|
---|
4158 | *
|
---|
4159 | * @returns VBox status code.
|
---|
4160 | * @param pGVM The global (ring-0) VM structure.
|
---|
4161 | * @param idCpu Cpu id.
|
---|
4162 | * @param pReq Pointer to the request packet.
|
---|
4163 | *
|
---|
4164 | * @thread EMT(idCpu)
|
---|
4165 | */
|
---|
4166 | GMMR0DECL(int) GMMR0QueryMemoryStatsReq(PGVM pGVM, VMCPUID idCpu, PGMMMEMSTATSREQ pReq)
|
---|
4167 | {
|
---|
4168 | /*
|
---|
4169 | * Validate input and pass it on.
|
---|
4170 | */
|
---|
4171 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
4172 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMMEMSTATSREQ),
|
---|
4173 | ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMMEMSTATSREQ)),
|
---|
4174 | VERR_INVALID_PARAMETER);
|
---|
4175 |
|
---|
4176 | /*
|
---|
4177 | * Validate input and get the basics.
|
---|
4178 | */
|
---|
4179 | PGMM pGMM;
|
---|
4180 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
4181 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
4182 | if (RT_FAILURE(rc))
|
---|
4183 | return rc;
|
---|
4184 |
|
---|
4185 | /*
|
---|
4186 | * Take the semaphore and do some more validations.
|
---|
4187 | */
|
---|
4188 | gmmR0MutexAcquire(pGMM);
|
---|
4189 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
4190 | {
|
---|
4191 | pReq->cAllocPages = pGVM->gmm.s.Stats.Allocated.cBasePages;
|
---|
4192 | pReq->cBalloonedPages = pGVM->gmm.s.Stats.cBalloonedPages;
|
---|
4193 | pReq->cMaxPages = pGVM->gmm.s.Stats.Reserved.cBasePages;
|
---|
4194 | pReq->cFreePages = pReq->cMaxPages - pReq->cAllocPages;
|
---|
4195 | }
|
---|
4196 | else
|
---|
4197 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
4198 |
|
---|
4199 | gmmR0MutexRelease(pGMM);
|
---|
4200 | LogFlow(("GMMR3QueryVMMemoryStats: returns %Rrc\n", rc));
|
---|
4201 | return rc;
|
---|
4202 | }
|
---|
4203 |
|
---|
4204 |
|
---|
4205 | /**
|
---|
4206 | * Worker for gmmR0UnmapChunk and gmmr0FreeChunk.
|
---|
4207 | *
|
---|
4208 | * Don't call this in legacy allocation mode!
|
---|
4209 | *
|
---|
4210 | * @returns VBox status code.
|
---|
4211 | * @param pGMM Pointer to the GMM instance data.
|
---|
4212 | * @param pGVM Pointer to the Global VM structure.
|
---|
4213 | * @param pChunk Pointer to the chunk to be unmapped.
|
---|
4214 | */
|
---|
4215 | static int gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
|
---|
4216 | {
|
---|
4217 | RT_NOREF_PV(pGMM);
|
---|
4218 |
|
---|
4219 | /*
|
---|
4220 | * Find the mapping and try unmapping it.
|
---|
4221 | */
|
---|
4222 | uint32_t cMappings = pChunk->cMappingsX;
|
---|
4223 | for (uint32_t i = 0; i < cMappings; i++)
|
---|
4224 | {
|
---|
4225 | Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ);
|
---|
4226 | if (pChunk->paMappingsX[i].pGVM == pGVM)
|
---|
4227 | {
|
---|
4228 | /* unmap */
|
---|
4229 | int rc = RTR0MemObjFree(pChunk->paMappingsX[i].hMapObj, false /* fFreeMappings (NA) */);
|
---|
4230 | if (RT_SUCCESS(rc))
|
---|
4231 | {
|
---|
4232 | /* update the record. */
|
---|
4233 | cMappings--;
|
---|
4234 | if (i < cMappings)
|
---|
4235 | pChunk->paMappingsX[i] = pChunk->paMappingsX[cMappings];
|
---|
4236 | pChunk->paMappingsX[cMappings].hMapObj = NIL_RTR0MEMOBJ;
|
---|
4237 | pChunk->paMappingsX[cMappings].pGVM = NULL;
|
---|
4238 | Assert(pChunk->cMappingsX - 1U == cMappings);
|
---|
4239 | pChunk->cMappingsX = cMappings;
|
---|
4240 | }
|
---|
4241 |
|
---|
4242 | return rc;
|
---|
4243 | }
|
---|
4244 | }
|
---|
4245 |
|
---|
4246 | Log(("gmmR0UnmapChunk: Chunk %#x is not mapped into pGVM=%p/%#x\n", pChunk->Core.Key, pGVM, pGVM->hSelf));
|
---|
4247 | return VERR_GMM_CHUNK_NOT_MAPPED;
|
---|
4248 | }
|
---|
4249 |
|
---|
4250 |
|
---|
4251 | /**
|
---|
4252 | * Unmaps a chunk previously mapped into the address space of the current process.
|
---|
4253 | *
|
---|
4254 | * @returns VBox status code.
|
---|
4255 | * @param pGMM Pointer to the GMM instance data.
|
---|
4256 | * @param pGVM Pointer to the Global VM structure.
|
---|
4257 | * @param pChunk Pointer to the chunk to be unmapped.
|
---|
4258 | * @param fRelaxedSem Whether we can release the semaphore while doing the
|
---|
4259 | * mapping (@c true) or not.
|
---|
4260 | */
|
---|
4261 | static int gmmR0UnmapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem)
|
---|
4262 | {
|
---|
4263 | /*
|
---|
4264 | * Lock the chunk and if possible leave the giant GMM lock.
|
---|
4265 | */
|
---|
4266 | GMMR0CHUNKMTXSTATE MtxState;
|
---|
4267 | int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk,
|
---|
4268 | fRelaxedSem ? GMMR0CHUNK_MTX_RETAKE_GIANT : GMMR0CHUNK_MTX_KEEP_GIANT);
|
---|
4269 | if (RT_SUCCESS(rc))
|
---|
4270 | {
|
---|
4271 | rc = gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk);
|
---|
4272 | gmmR0ChunkMutexRelease(&MtxState, pChunk);
|
---|
4273 | }
|
---|
4274 | return rc;
|
---|
4275 | }
|
---|
4276 |
|
---|
4277 |
|
---|
4278 | /**
|
---|
4279 | * Worker for gmmR0MapChunk.
|
---|
4280 | *
|
---|
4281 | * @returns VBox status code.
|
---|
4282 | * @param pGMM Pointer to the GMM instance data.
|
---|
4283 | * @param pGVM Pointer to the Global VM structure.
|
---|
4284 | * @param pChunk Pointer to the chunk to be mapped.
|
---|
4285 | * @param ppvR3 Where to store the ring-3 address of the mapping.
|
---|
4286 | * In the VERR_GMM_CHUNK_ALREADY_MAPPED case, this will be
|
---|
4287 | * contain the address of the existing mapping.
|
---|
4288 | */
|
---|
4289 | static int gmmR0MapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3)
|
---|
4290 | {
|
---|
4291 | RT_NOREF(pGMM);
|
---|
4292 |
|
---|
4293 | /*
|
---|
4294 | * Check to see if the chunk is already mapped.
|
---|
4295 | */
|
---|
4296 | for (uint32_t i = 0; i < pChunk->cMappingsX; i++)
|
---|
4297 | {
|
---|
4298 | Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ);
|
---|
4299 | if (pChunk->paMappingsX[i].pGVM == pGVM)
|
---|
4300 | {
|
---|
4301 | *ppvR3 = RTR0MemObjAddressR3(pChunk->paMappingsX[i].hMapObj);
|
---|
4302 | Log(("gmmR0MapChunk: chunk %#x is already mapped at %p!\n", pChunk->Core.Key, *ppvR3));
|
---|
4303 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
4304 | /* The ring-3 chunk cache can be out of sync; don't fail. */
|
---|
4305 | return VINF_SUCCESS;
|
---|
4306 | #else
|
---|
4307 | return VERR_GMM_CHUNK_ALREADY_MAPPED;
|
---|
4308 | #endif
|
---|
4309 | }
|
---|
4310 | }
|
---|
4311 |
|
---|
4312 | /*
|
---|
4313 | * Do the mapping.
|
---|
4314 | */
|
---|
4315 | RTR0MEMOBJ hMapObj;
|
---|
4316 | int rc = RTR0MemObjMapUser(&hMapObj, pChunk->hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
|
---|
4317 | if (RT_SUCCESS(rc))
|
---|
4318 | {
|
---|
4319 | /* reallocate the array? assumes few users per chunk (usually one). */
|
---|
4320 | unsigned iMapping = pChunk->cMappingsX;
|
---|
4321 | if ( iMapping <= 3
|
---|
4322 | || (iMapping & 3) == 0)
|
---|
4323 | {
|
---|
4324 | unsigned cNewSize = iMapping <= 3
|
---|
4325 | ? iMapping + 1
|
---|
4326 | : iMapping + 4;
|
---|
4327 | Assert(cNewSize < 4 || RT_ALIGN_32(cNewSize, 4) == cNewSize);
|
---|
4328 | if (RT_UNLIKELY(cNewSize > UINT16_MAX))
|
---|
4329 | {
|
---|
4330 | rc = RTR0MemObjFree(hMapObj, false /* fFreeMappings (NA) */); AssertRC(rc);
|
---|
4331 | return VERR_GMM_TOO_MANY_CHUNK_MAPPINGS;
|
---|
4332 | }
|
---|
4333 |
|
---|
4334 | void *pvMappings = RTMemRealloc(pChunk->paMappingsX, cNewSize * sizeof(pChunk->paMappingsX[0]));
|
---|
4335 | if (RT_UNLIKELY(!pvMappings))
|
---|
4336 | {
|
---|
4337 | rc = RTR0MemObjFree(hMapObj, false /* fFreeMappings (NA) */); AssertRC(rc);
|
---|
4338 | return VERR_NO_MEMORY;
|
---|
4339 | }
|
---|
4340 | pChunk->paMappingsX = (PGMMCHUNKMAP)pvMappings;
|
---|
4341 | }
|
---|
4342 |
|
---|
4343 | /* insert new entry */
|
---|
4344 | pChunk->paMappingsX[iMapping].hMapObj = hMapObj;
|
---|
4345 | pChunk->paMappingsX[iMapping].pGVM = pGVM;
|
---|
4346 | Assert(pChunk->cMappingsX == iMapping);
|
---|
4347 | pChunk->cMappingsX = iMapping + 1;
|
---|
4348 |
|
---|
4349 | *ppvR3 = RTR0MemObjAddressR3(hMapObj);
|
---|
4350 | }
|
---|
4351 |
|
---|
4352 | return rc;
|
---|
4353 | }
|
---|
4354 |
|
---|
4355 |
|
---|
4356 | /**
|
---|
4357 | * Maps a chunk into the user address space of the current process.
|
---|
4358 | *
|
---|
4359 | * @returns VBox status code.
|
---|
4360 | * @param pGMM Pointer to the GMM instance data.
|
---|
4361 | * @param pGVM Pointer to the Global VM structure.
|
---|
4362 | * @param pChunk Pointer to the chunk to be mapped.
|
---|
4363 | * @param fRelaxedSem Whether we can release the semaphore while doing the
|
---|
4364 | * mapping (@c true) or not.
|
---|
4365 | * @param ppvR3 Where to store the ring-3 address of the mapping.
|
---|
4366 | * In the VERR_GMM_CHUNK_ALREADY_MAPPED case, this will be
|
---|
4367 | * contain the address of the existing mapping.
|
---|
4368 | */
|
---|
4369 | static int gmmR0MapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem, PRTR3PTR ppvR3)
|
---|
4370 | {
|
---|
4371 | /*
|
---|
4372 | * Take the chunk lock and leave the giant GMM lock when possible, then
|
---|
4373 | * call the worker function.
|
---|
4374 | */
|
---|
4375 | GMMR0CHUNKMTXSTATE MtxState;
|
---|
4376 | int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk,
|
---|
4377 | fRelaxedSem ? GMMR0CHUNK_MTX_RETAKE_GIANT : GMMR0CHUNK_MTX_KEEP_GIANT);
|
---|
4378 | if (RT_SUCCESS(rc))
|
---|
4379 | {
|
---|
4380 | rc = gmmR0MapChunkLocked(pGMM, pGVM, pChunk, ppvR3);
|
---|
4381 | gmmR0ChunkMutexRelease(&MtxState, pChunk);
|
---|
4382 | }
|
---|
4383 |
|
---|
4384 | return rc;
|
---|
4385 | }
|
---|
4386 |
|
---|
4387 |
|
---|
4388 |
|
---|
4389 | #if defined(VBOX_WITH_PAGE_SHARING) || defined(VBOX_STRICT)
|
---|
4390 | /**
|
---|
4391 | * Check if a chunk is mapped into the specified VM
|
---|
4392 | *
|
---|
4393 | * @returns mapped yes/no
|
---|
4394 | * @param pGMM Pointer to the GMM instance.
|
---|
4395 | * @param pGVM Pointer to the Global VM structure.
|
---|
4396 | * @param pChunk Pointer to the chunk to be mapped.
|
---|
4397 | * @param ppvR3 Where to store the ring-3 address of the mapping.
|
---|
4398 | */
|
---|
4399 | static bool gmmR0IsChunkMapped(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3)
|
---|
4400 | {
|
---|
4401 | GMMR0CHUNKMTXSTATE MtxState;
|
---|
4402 | gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
|
---|
4403 | for (uint32_t i = 0; i < pChunk->cMappingsX; i++)
|
---|
4404 | {
|
---|
4405 | Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ);
|
---|
4406 | if (pChunk->paMappingsX[i].pGVM == pGVM)
|
---|
4407 | {
|
---|
4408 | *ppvR3 = RTR0MemObjAddressR3(pChunk->paMappingsX[i].hMapObj);
|
---|
4409 | gmmR0ChunkMutexRelease(&MtxState, pChunk);
|
---|
4410 | return true;
|
---|
4411 | }
|
---|
4412 | }
|
---|
4413 | *ppvR3 = NULL;
|
---|
4414 | gmmR0ChunkMutexRelease(&MtxState, pChunk);
|
---|
4415 | return false;
|
---|
4416 | }
|
---|
4417 | #endif /* VBOX_WITH_PAGE_SHARING || VBOX_STRICT */
|
---|
4418 |
|
---|
4419 |
|
---|
4420 | /**
|
---|
4421 | * Map a chunk and/or unmap another chunk.
|
---|
4422 | *
|
---|
4423 | * The mapping and unmapping applies to the current process.
|
---|
4424 | *
|
---|
4425 | * This API does two things because it saves a kernel call per mapping when
|
---|
4426 | * when the ring-3 mapping cache is full.
|
---|
4427 | *
|
---|
4428 | * @returns VBox status code.
|
---|
4429 | * @param pGVM The global (ring-0) VM structure.
|
---|
4430 | * @param idChunkMap The chunk to map. NIL_GMM_CHUNKID if nothing to map.
|
---|
4431 | * @param idChunkUnmap The chunk to unmap. NIL_GMM_CHUNKID if nothing to unmap.
|
---|
4432 | * @param ppvR3 Where to store the address of the mapped chunk. NULL is ok if nothing to map.
|
---|
4433 | * @thread EMT ???
|
---|
4434 | */
|
---|
4435 | GMMR0DECL(int) GMMR0MapUnmapChunk(PGVM pGVM, uint32_t idChunkMap, uint32_t idChunkUnmap, PRTR3PTR ppvR3)
|
---|
4436 | {
|
---|
4437 | LogFlow(("GMMR0MapUnmapChunk: pGVM=%p idChunkMap=%#x idChunkUnmap=%#x ppvR3=%p\n",
|
---|
4438 | pGVM, idChunkMap, idChunkUnmap, ppvR3));
|
---|
4439 |
|
---|
4440 | /*
|
---|
4441 | * Validate input and get the basics.
|
---|
4442 | */
|
---|
4443 | PGMM pGMM;
|
---|
4444 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
4445 | int rc = GVMMR0ValidateGVM(pGVM);
|
---|
4446 | if (RT_FAILURE(rc))
|
---|
4447 | return rc;
|
---|
4448 |
|
---|
4449 | AssertCompile(NIL_GMM_CHUNKID == 0);
|
---|
4450 | AssertMsgReturn(idChunkMap <= GMM_CHUNKID_LAST, ("%#x\n", idChunkMap), VERR_INVALID_PARAMETER);
|
---|
4451 | AssertMsgReturn(idChunkUnmap <= GMM_CHUNKID_LAST, ("%#x\n", idChunkUnmap), VERR_INVALID_PARAMETER);
|
---|
4452 |
|
---|
4453 | if ( idChunkMap == NIL_GMM_CHUNKID
|
---|
4454 | && idChunkUnmap == NIL_GMM_CHUNKID)
|
---|
4455 | return VERR_INVALID_PARAMETER;
|
---|
4456 |
|
---|
4457 | if (idChunkMap != NIL_GMM_CHUNKID)
|
---|
4458 | {
|
---|
4459 | AssertPtrReturn(ppvR3, VERR_INVALID_POINTER);
|
---|
4460 | *ppvR3 = NIL_RTR3PTR;
|
---|
4461 | }
|
---|
4462 |
|
---|
4463 | /*
|
---|
4464 | * Take the semaphore and do the work.
|
---|
4465 | *
|
---|
4466 | * The unmapping is done last since it's easier to undo a mapping than
|
---|
4467 | * undoing an unmapping. The ring-3 mapping cache cannot not be so big
|
---|
4468 | * that it pushes the user virtual address space to within a chunk of
|
---|
4469 | * it it's limits, so, no problem here.
|
---|
4470 | */
|
---|
4471 | gmmR0MutexAcquire(pGMM);
|
---|
4472 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
4473 | {
|
---|
4474 | PGMMCHUNK pMap = NULL;
|
---|
4475 | if (idChunkMap != NIL_GVM_HANDLE)
|
---|
4476 | {
|
---|
4477 | pMap = gmmR0GetChunk(pGMM, idChunkMap);
|
---|
4478 | if (RT_LIKELY(pMap))
|
---|
4479 | rc = gmmR0MapChunk(pGMM, pGVM, pMap, true /*fRelaxedSem*/, ppvR3);
|
---|
4480 | else
|
---|
4481 | {
|
---|
4482 | Log(("GMMR0MapUnmapChunk: idChunkMap=%#x\n", idChunkMap));
|
---|
4483 | rc = VERR_GMM_CHUNK_NOT_FOUND;
|
---|
4484 | }
|
---|
4485 | }
|
---|
4486 | /** @todo split this operation, the bail out might (theoretcially) not be
|
---|
4487 | * entirely safe. */
|
---|
4488 |
|
---|
4489 | if ( idChunkUnmap != NIL_GMM_CHUNKID
|
---|
4490 | && RT_SUCCESS(rc))
|
---|
4491 | {
|
---|
4492 | PGMMCHUNK pUnmap = gmmR0GetChunk(pGMM, idChunkUnmap);
|
---|
4493 | if (RT_LIKELY(pUnmap))
|
---|
4494 | rc = gmmR0UnmapChunk(pGMM, pGVM, pUnmap, true /*fRelaxedSem*/);
|
---|
4495 | else
|
---|
4496 | {
|
---|
4497 | Log(("GMMR0MapUnmapChunk: idChunkUnmap=%#x\n", idChunkUnmap));
|
---|
4498 | rc = VERR_GMM_CHUNK_NOT_FOUND;
|
---|
4499 | }
|
---|
4500 |
|
---|
4501 | if (RT_FAILURE(rc) && pMap)
|
---|
4502 | gmmR0UnmapChunk(pGMM, pGVM, pMap, false /*fRelaxedSem*/);
|
---|
4503 | }
|
---|
4504 |
|
---|
4505 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
4506 | }
|
---|
4507 | else
|
---|
4508 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
4509 | gmmR0MutexRelease(pGMM);
|
---|
4510 |
|
---|
4511 | LogFlow(("GMMR0MapUnmapChunk: returns %Rrc\n", rc));
|
---|
4512 | return rc;
|
---|
4513 | }
|
---|
4514 |
|
---|
4515 |
|
---|
4516 | /**
|
---|
4517 | * VMMR0 request wrapper for GMMR0MapUnmapChunk.
|
---|
4518 | *
|
---|
4519 | * @returns see GMMR0MapUnmapChunk.
|
---|
4520 | * @param pGVM The global (ring-0) VM structure.
|
---|
4521 | * @param pReq Pointer to the request packet.
|
---|
4522 | */
|
---|
4523 | GMMR0DECL(int) GMMR0MapUnmapChunkReq(PGVM pGVM, PGMMMAPUNMAPCHUNKREQ pReq)
|
---|
4524 | {
|
---|
4525 | /*
|
---|
4526 | * Validate input and pass it on.
|
---|
4527 | */
|
---|
4528 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
4529 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
|
---|
4530 |
|
---|
4531 | return GMMR0MapUnmapChunk(pGVM, pReq->idChunkMap, pReq->idChunkUnmap, &pReq->pvR3);
|
---|
4532 | }
|
---|
4533 |
|
---|
4534 |
|
---|
4535 | #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
|
---|
4536 | /**
|
---|
4537 | * Gets the ring-0 virtual address for the given page.
|
---|
4538 | *
|
---|
4539 | * This is used by PGM when IEM and such wants to access guest RAM from ring-0.
|
---|
4540 | * One of the ASSUMPTIONS here is that the @a idPage is used by the VM and the
|
---|
4541 | * corresponding chunk will remain valid beyond the call (at least till the EMT
|
---|
4542 | * returns to ring-3).
|
---|
4543 | *
|
---|
4544 | * @returns VBox status code.
|
---|
4545 | * @param pGVM Pointer to the kernel-only VM instace data.
|
---|
4546 | * @param idPage The page ID.
|
---|
4547 | * @param ppv Where to store the address.
|
---|
4548 | * @thread EMT
|
---|
4549 | */
|
---|
4550 | GMMR0DECL(int) GMMR0PageIdToVirt(PGVM pGVM, uint32_t idPage, void **ppv)
|
---|
4551 | {
|
---|
4552 | *ppv = NULL;
|
---|
4553 | PGMM pGMM;
|
---|
4554 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
4555 |
|
---|
4556 | uint32_t const idChunk = idPage >> GMM_CHUNKID_SHIFT;
|
---|
4557 |
|
---|
4558 | /*
|
---|
4559 | * Start with the per-VM TLB.
|
---|
4560 | */
|
---|
4561 | RTSpinlockAcquire(pGVM->gmm.s.hChunkTlbSpinLock);
|
---|
4562 |
|
---|
4563 | PGMMPERVMCHUNKTLBE pTlbe = &pGVM->gmm.s.aChunkTlbEntries[GMMPERVM_CHUNKTLB_IDX(idChunk)];
|
---|
4564 | PGMMCHUNK pChunk = pTlbe->pChunk;
|
---|
4565 | if ( pChunk != NULL
|
---|
4566 | && pTlbe->idGeneration == ASMAtomicUoReadU64(&pGMM->idFreeGeneration)
|
---|
4567 | && pChunk->Core.Key == idChunk)
|
---|
4568 | pGVM->R0Stats.gmm.cChunkTlbHits++; /* hopefully this is a likely outcome */
|
---|
4569 | else
|
---|
4570 | {
|
---|
4571 | pGVM->R0Stats.gmm.cChunkTlbMisses++;
|
---|
4572 |
|
---|
4573 | /*
|
---|
4574 | * Look it up in the chunk tree.
|
---|
4575 | */
|
---|
4576 | RTSpinlockAcquire(pGMM->hSpinLockTree);
|
---|
4577 | pChunk = gmmR0GetChunkLocked(pGMM, idChunk);
|
---|
4578 | if (RT_LIKELY(pChunk))
|
---|
4579 | {
|
---|
4580 | pTlbe->idGeneration = pGMM->idFreeGeneration;
|
---|
4581 | RTSpinlockRelease(pGMM->hSpinLockTree);
|
---|
4582 | pTlbe->pChunk = pChunk;
|
---|
4583 | }
|
---|
4584 | else
|
---|
4585 | {
|
---|
4586 | RTSpinlockRelease(pGMM->hSpinLockTree);
|
---|
4587 | RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock);
|
---|
4588 | AssertMsgFailed(("idPage=%#x\n", idPage));
|
---|
4589 | return VERR_GMM_PAGE_NOT_FOUND;
|
---|
4590 | }
|
---|
4591 | }
|
---|
4592 |
|
---|
4593 | RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock);
|
---|
4594 |
|
---|
4595 | /*
|
---|
4596 | * Got a chunk, now validate the page ownership and calcuate it's address.
|
---|
4597 | */
|
---|
4598 | const GMMPAGE * const pPage = &pChunk->aPages[idPage & GMM_PAGEID_IDX_MASK];
|
---|
4599 | if (RT_LIKELY( ( GMM_PAGE_IS_PRIVATE(pPage)
|
---|
4600 | && pPage->Private.hGVM == pGVM->hSelf)
|
---|
4601 | || GMM_PAGE_IS_SHARED(pPage)))
|
---|
4602 | {
|
---|
4603 | AssertPtr(pChunk->pbMapping);
|
---|
4604 | *ppv = &pChunk->pbMapping[(idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT];
|
---|
4605 | return VINF_SUCCESS;
|
---|
4606 | }
|
---|
4607 | AssertMsgFailed(("idPage=%#x is-private=%RTbool Private.hGVM=%u pGVM->hGVM=%u\n",
|
---|
4608 | idPage, GMM_PAGE_IS_PRIVATE(pPage), pPage->Private.hGVM, pGVM->hSelf));
|
---|
4609 | return VERR_GMM_NOT_PAGE_OWNER;
|
---|
4610 | }
|
---|
4611 | #endif /* !VBOX_WITH_LINEAR_HOST_PHYS_MEM */
|
---|
4612 |
|
---|
4613 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
4614 |
|
---|
4615 | # ifdef VBOX_STRICT
|
---|
4616 | /**
|
---|
4617 | * For checksumming shared pages in strict builds.
|
---|
4618 | *
|
---|
4619 | * The purpose is making sure that a page doesn't change.
|
---|
4620 | *
|
---|
4621 | * @returns Checksum, 0 on failure.
|
---|
4622 | * @param pGMM The GMM instance data.
|
---|
4623 | * @param pGVM Pointer to the kernel-only VM instace data.
|
---|
4624 | * @param idPage The page ID.
|
---|
4625 | */
|
---|
4626 | static uint32_t gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage)
|
---|
4627 | {
|
---|
4628 | PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
|
---|
4629 | AssertMsgReturn(pChunk, ("idPage=%#x\n", idPage), 0);
|
---|
4630 |
|
---|
4631 | uint8_t *pbChunk;
|
---|
4632 | if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
|
---|
4633 | return 0;
|
---|
4634 | uint8_t const *pbPage = pbChunk + ((idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
|
---|
4635 |
|
---|
4636 | return RTCrc32(pbPage, GUEST_PAGE_SIZE);
|
---|
4637 | }
|
---|
4638 | # endif /* VBOX_STRICT */
|
---|
4639 |
|
---|
4640 |
|
---|
4641 | /**
|
---|
4642 | * Calculates the module hash value.
|
---|
4643 | *
|
---|
4644 | * @returns Hash value.
|
---|
4645 | * @param pszModuleName The module name.
|
---|
4646 | * @param pszVersion The module version string.
|
---|
4647 | */
|
---|
4648 | static uint32_t gmmR0ShModCalcHash(const char *pszModuleName, const char *pszVersion)
|
---|
4649 | {
|
---|
4650 | return RTStrHash1ExN(3, pszModuleName, RTSTR_MAX, "::", (size_t)2, pszVersion, RTSTR_MAX);
|
---|
4651 | }
|
---|
4652 |
|
---|
4653 |
|
---|
4654 | /**
|
---|
4655 | * Finds a global module.
|
---|
4656 | *
|
---|
4657 | * @returns Pointer to the global module on success, NULL if not found.
|
---|
4658 | * @param pGMM The GMM instance data.
|
---|
4659 | * @param uHash The hash as calculated by gmmR0ShModCalcHash.
|
---|
4660 | * @param cbModule The module size.
|
---|
4661 | * @param enmGuestOS The guest OS type.
|
---|
4662 | * @param cRegions The number of regions.
|
---|
4663 | * @param pszModuleName The module name.
|
---|
4664 | * @param pszVersion The module version.
|
---|
4665 | * @param paRegions The region descriptions.
|
---|
4666 | */
|
---|
4667 | static PGMMSHAREDMODULE gmmR0ShModFindGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS,
|
---|
4668 | uint32_t cRegions, const char *pszModuleName, const char *pszVersion,
|
---|
4669 | struct VMMDEVSHAREDREGIONDESC const *paRegions)
|
---|
4670 | {
|
---|
4671 | for (PGMMSHAREDMODULE pGblMod = (PGMMSHAREDMODULE)RTAvllU32Get(&pGMM->pGlobalSharedModuleTree, uHash);
|
---|
4672 | pGblMod;
|
---|
4673 | pGblMod = (PGMMSHAREDMODULE)pGblMod->Core.pList)
|
---|
4674 | {
|
---|
4675 | if (pGblMod->cbModule != cbModule)
|
---|
4676 | continue;
|
---|
4677 | if (pGblMod->enmGuestOS != enmGuestOS)
|
---|
4678 | continue;
|
---|
4679 | if (pGblMod->cRegions != cRegions)
|
---|
4680 | continue;
|
---|
4681 | if (strcmp(pGblMod->szName, pszModuleName))
|
---|
4682 | continue;
|
---|
4683 | if (strcmp(pGblMod->szVersion, pszVersion))
|
---|
4684 | continue;
|
---|
4685 |
|
---|
4686 | uint32_t i;
|
---|
4687 | for (i = 0; i < cRegions; i++)
|
---|
4688 | {
|
---|
4689 | uint32_t off = paRegions[i].GCRegionAddr & GUEST_PAGE_OFFSET_MASK;
|
---|
4690 | if (pGblMod->aRegions[i].off != off)
|
---|
4691 | break;
|
---|
4692 |
|
---|
4693 | uint32_t cb = RT_ALIGN_32(paRegions[i].cbRegion + off, GUEST_PAGE_SIZE);
|
---|
4694 | if (pGblMod->aRegions[i].cb != cb)
|
---|
4695 | break;
|
---|
4696 | }
|
---|
4697 |
|
---|
4698 | if (i == cRegions)
|
---|
4699 | return pGblMod;
|
---|
4700 | }
|
---|
4701 |
|
---|
4702 | return NULL;
|
---|
4703 | }
|
---|
4704 |
|
---|
4705 |
|
---|
4706 | /**
|
---|
4707 | * Creates a new global module.
|
---|
4708 | *
|
---|
4709 | * @returns VBox status code.
|
---|
4710 | * @param pGMM The GMM instance data.
|
---|
4711 | * @param uHash The hash as calculated by gmmR0ShModCalcHash.
|
---|
4712 | * @param cbModule The module size.
|
---|
4713 | * @param enmGuestOS The guest OS type.
|
---|
4714 | * @param cRegions The number of regions.
|
---|
4715 | * @param pszModuleName The module name.
|
---|
4716 | * @param pszVersion The module version.
|
---|
4717 | * @param paRegions The region descriptions.
|
---|
4718 | * @param ppGblMod Where to return the new module on success.
|
---|
4719 | */
|
---|
4720 | static int gmmR0ShModNewGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS,
|
---|
4721 | uint32_t cRegions, const char *pszModuleName, const char *pszVersion,
|
---|
4722 | struct VMMDEVSHAREDREGIONDESC const *paRegions, PGMMSHAREDMODULE *ppGblMod)
|
---|
4723 | {
|
---|
4724 | Log(("gmmR0ShModNewGlobal: %s %s size %#x os %u rgn %u\n", pszModuleName, pszVersion, cbModule, enmGuestOS, cRegions));
|
---|
4725 | if (pGMM->cShareableModules >= GMM_MAX_SHARED_GLOBAL_MODULES)
|
---|
4726 | {
|
---|
4727 | Log(("gmmR0ShModNewGlobal: Too many modules\n"));
|
---|
4728 | return VERR_GMM_TOO_MANY_GLOBAL_MODULES;
|
---|
4729 | }
|
---|
4730 |
|
---|
4731 | PGMMSHAREDMODULE pGblMod = (PGMMSHAREDMODULE)RTMemAllocZ(RT_UOFFSETOF_DYN(GMMSHAREDMODULE, aRegions[cRegions]));
|
---|
4732 | if (!pGblMod)
|
---|
4733 | {
|
---|
4734 | Log(("gmmR0ShModNewGlobal: No memory\n"));
|
---|
4735 | return VERR_NO_MEMORY;
|
---|
4736 | }
|
---|
4737 |
|
---|
4738 | pGblMod->Core.Key = uHash;
|
---|
4739 | pGblMod->cbModule = cbModule;
|
---|
4740 | pGblMod->cRegions = cRegions;
|
---|
4741 | pGblMod->cUsers = 1;
|
---|
4742 | pGblMod->enmGuestOS = enmGuestOS;
|
---|
4743 | strcpy(pGblMod->szName, pszModuleName);
|
---|
4744 | strcpy(pGblMod->szVersion, pszVersion);
|
---|
4745 |
|
---|
4746 | for (uint32_t i = 0; i < cRegions; i++)
|
---|
4747 | {
|
---|
4748 | Log(("gmmR0ShModNewGlobal: rgn[%u]=%RGvLB%#x\n", i, paRegions[i].GCRegionAddr, paRegions[i].cbRegion));
|
---|
4749 | pGblMod->aRegions[i].off = paRegions[i].GCRegionAddr & GUEST_PAGE_OFFSET_MASK;
|
---|
4750 | pGblMod->aRegions[i].cb = paRegions[i].cbRegion + pGblMod->aRegions[i].off;
|
---|
4751 | pGblMod->aRegions[i].cb = RT_ALIGN_32(pGblMod->aRegions[i].cb, GUEST_PAGE_SIZE);
|
---|
4752 | pGblMod->aRegions[i].paidPages = NULL; /* allocated when needed. */
|
---|
4753 | }
|
---|
4754 |
|
---|
4755 | bool fInsert = RTAvllU32Insert(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core);
|
---|
4756 | Assert(fInsert); NOREF(fInsert);
|
---|
4757 | pGMM->cShareableModules++;
|
---|
4758 |
|
---|
4759 | *ppGblMod = pGblMod;
|
---|
4760 | return VINF_SUCCESS;
|
---|
4761 | }
|
---|
4762 |
|
---|
4763 |
|
---|
4764 | /**
|
---|
4765 | * Deletes a global module which is no longer referenced by anyone.
|
---|
4766 | *
|
---|
4767 | * @param pGMM The GMM instance data.
|
---|
4768 | * @param pGblMod The module to delete.
|
---|
4769 | */
|
---|
4770 | static void gmmR0ShModDeleteGlobal(PGMM pGMM, PGMMSHAREDMODULE pGblMod)
|
---|
4771 | {
|
---|
4772 | Assert(pGblMod->cUsers == 0);
|
---|
4773 | Assert(pGMM->cShareableModules > 0 && pGMM->cShareableModules <= GMM_MAX_SHARED_GLOBAL_MODULES);
|
---|
4774 |
|
---|
4775 | void *pvTest = RTAvllU32RemoveNode(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core);
|
---|
4776 | Assert(pvTest == pGblMod); NOREF(pvTest);
|
---|
4777 | pGMM->cShareableModules--;
|
---|
4778 |
|
---|
4779 | uint32_t i = pGblMod->cRegions;
|
---|
4780 | while (i-- > 0)
|
---|
4781 | {
|
---|
4782 | if (pGblMod->aRegions[i].paidPages)
|
---|
4783 | {
|
---|
4784 | /* We don't doing anything to the pages as they are handled by the
|
---|
4785 | copy-on-write mechanism in PGM. */
|
---|
4786 | RTMemFree(pGblMod->aRegions[i].paidPages);
|
---|
4787 | pGblMod->aRegions[i].paidPages = NULL;
|
---|
4788 | }
|
---|
4789 | }
|
---|
4790 | RTMemFree(pGblMod);
|
---|
4791 | }
|
---|
4792 |
|
---|
4793 |
|
---|
4794 | static int gmmR0ShModNewPerVM(PGVM pGVM, RTGCPTR GCBaseAddr, uint32_t cRegions, const VMMDEVSHAREDREGIONDESC *paRegions,
|
---|
4795 | PGMMSHAREDMODULEPERVM *ppRecVM)
|
---|
4796 | {
|
---|
4797 | if (pGVM->gmm.s.Stats.cShareableModules >= GMM_MAX_SHARED_PER_VM_MODULES)
|
---|
4798 | return VERR_GMM_TOO_MANY_PER_VM_MODULES;
|
---|
4799 |
|
---|
4800 | PGMMSHAREDMODULEPERVM pRecVM;
|
---|
4801 | pRecVM = (PGMMSHAREDMODULEPERVM)RTMemAllocZ(RT_UOFFSETOF_DYN(GMMSHAREDMODULEPERVM, aRegionsGCPtrs[cRegions]));
|
---|
4802 | if (!pRecVM)
|
---|
4803 | return VERR_NO_MEMORY;
|
---|
4804 |
|
---|
4805 | pRecVM->Core.Key = GCBaseAddr;
|
---|
4806 | for (uint32_t i = 0; i < cRegions; i++)
|
---|
4807 | pRecVM->aRegionsGCPtrs[i] = paRegions[i].GCRegionAddr;
|
---|
4808 |
|
---|
4809 | bool fInsert = RTAvlGCPtrInsert(&pGVM->gmm.s.pSharedModuleTree, &pRecVM->Core);
|
---|
4810 | Assert(fInsert); NOREF(fInsert);
|
---|
4811 | pGVM->gmm.s.Stats.cShareableModules++;
|
---|
4812 |
|
---|
4813 | *ppRecVM = pRecVM;
|
---|
4814 | return VINF_SUCCESS;
|
---|
4815 | }
|
---|
4816 |
|
---|
4817 |
|
---|
4818 | static void gmmR0ShModDeletePerVM(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULEPERVM pRecVM, bool fRemove)
|
---|
4819 | {
|
---|
4820 | /*
|
---|
4821 | * Free the per-VM module.
|
---|
4822 | */
|
---|
4823 | PGMMSHAREDMODULE pGblMod = pRecVM->pGlobalModule;
|
---|
4824 | pRecVM->pGlobalModule = NULL;
|
---|
4825 |
|
---|
4826 | if (fRemove)
|
---|
4827 | {
|
---|
4828 | void *pvTest = RTAvlGCPtrRemove(&pGVM->gmm.s.pSharedModuleTree, pRecVM->Core.Key);
|
---|
4829 | Assert(pvTest == &pRecVM->Core); NOREF(pvTest);
|
---|
4830 | }
|
---|
4831 |
|
---|
4832 | RTMemFree(pRecVM);
|
---|
4833 |
|
---|
4834 | /*
|
---|
4835 | * Release the global module.
|
---|
4836 | * (In the registration bailout case, it might not be.)
|
---|
4837 | */
|
---|
4838 | if (pGblMod)
|
---|
4839 | {
|
---|
4840 | Assert(pGblMod->cUsers > 0);
|
---|
4841 | pGblMod->cUsers--;
|
---|
4842 | if (pGblMod->cUsers == 0)
|
---|
4843 | gmmR0ShModDeleteGlobal(pGMM, pGblMod);
|
---|
4844 | }
|
---|
4845 | }
|
---|
4846 |
|
---|
4847 | #endif /* VBOX_WITH_PAGE_SHARING */
|
---|
4848 |
|
---|
4849 | /**
|
---|
4850 | * Registers a new shared module for the VM.
|
---|
4851 | *
|
---|
4852 | * @returns VBox status code.
|
---|
4853 | * @param pGVM The global (ring-0) VM structure.
|
---|
4854 | * @param idCpu The VCPU id.
|
---|
4855 | * @param enmGuestOS The guest OS type.
|
---|
4856 | * @param pszModuleName The module name.
|
---|
4857 | * @param pszVersion The module version.
|
---|
4858 | * @param GCPtrModBase The module base address.
|
---|
4859 | * @param cbModule The module size.
|
---|
4860 | * @param cRegions The mumber of shared region descriptors.
|
---|
4861 | * @param paRegions Pointer to an array of shared region(s).
|
---|
4862 | * @thread EMT(idCpu)
|
---|
4863 | */
|
---|
4864 | GMMR0DECL(int) GMMR0RegisterSharedModule(PGVM pGVM, VMCPUID idCpu, VBOXOSFAMILY enmGuestOS, char *pszModuleName,
|
---|
4865 | char *pszVersion, RTGCPTR GCPtrModBase, uint32_t cbModule,
|
---|
4866 | uint32_t cRegions, struct VMMDEVSHAREDREGIONDESC const *paRegions)
|
---|
4867 | {
|
---|
4868 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
4869 | /*
|
---|
4870 | * Validate input and get the basics.
|
---|
4871 | *
|
---|
4872 | * Note! Turns out the module size does necessarily match the size of the
|
---|
4873 | * regions. (iTunes on XP)
|
---|
4874 | */
|
---|
4875 | PGMM pGMM;
|
---|
4876 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
4877 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
4878 | if (RT_FAILURE(rc))
|
---|
4879 | return rc;
|
---|
4880 |
|
---|
4881 | if (RT_UNLIKELY(cRegions > VMMDEVSHAREDREGIONDESC_MAX))
|
---|
4882 | return VERR_GMM_TOO_MANY_REGIONS;
|
---|
4883 |
|
---|
4884 | if (RT_UNLIKELY(cbModule == 0 || cbModule > _1G))
|
---|
4885 | return VERR_GMM_BAD_SHARED_MODULE_SIZE;
|
---|
4886 |
|
---|
4887 | uint32_t cbTotal = 0;
|
---|
4888 | for (uint32_t i = 0; i < cRegions; i++)
|
---|
4889 | {
|
---|
4890 | if (RT_UNLIKELY(paRegions[i].cbRegion == 0 || paRegions[i].cbRegion > _1G))
|
---|
4891 | return VERR_GMM_SHARED_MODULE_BAD_REGIONS_SIZE;
|
---|
4892 |
|
---|
4893 | cbTotal += paRegions[i].cbRegion;
|
---|
4894 | if (RT_UNLIKELY(cbTotal > _1G))
|
---|
4895 | return VERR_GMM_SHARED_MODULE_BAD_REGIONS_SIZE;
|
---|
4896 | }
|
---|
4897 |
|
---|
4898 | AssertPtrReturn(pszModuleName, VERR_INVALID_POINTER);
|
---|
4899 | if (RT_UNLIKELY(!memchr(pszModuleName, '\0', GMM_SHARED_MODULE_MAX_NAME_STRING)))
|
---|
4900 | return VERR_GMM_MODULE_NAME_TOO_LONG;
|
---|
4901 |
|
---|
4902 | AssertPtrReturn(pszVersion, VERR_INVALID_POINTER);
|
---|
4903 | if (RT_UNLIKELY(!memchr(pszVersion, '\0', GMM_SHARED_MODULE_MAX_VERSION_STRING)))
|
---|
4904 | return VERR_GMM_MODULE_NAME_TOO_LONG;
|
---|
4905 |
|
---|
4906 | uint32_t const uHash = gmmR0ShModCalcHash(pszModuleName, pszVersion);
|
---|
4907 | Log(("GMMR0RegisterSharedModule %s %s base %RGv size %x hash %x\n", pszModuleName, pszVersion, GCPtrModBase, cbModule, uHash));
|
---|
4908 |
|
---|
4909 | /*
|
---|
4910 | * Take the semaphore and do some more validations.
|
---|
4911 | */
|
---|
4912 | gmmR0MutexAcquire(pGMM);
|
---|
4913 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
4914 | {
|
---|
4915 | /*
|
---|
4916 | * Check if this module is already locally registered and register
|
---|
4917 | * it if it isn't. The base address is a unique module identifier
|
---|
4918 | * locally.
|
---|
4919 | */
|
---|
4920 | PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)RTAvlGCPtrGet(&pGVM->gmm.s.pSharedModuleTree, GCPtrModBase);
|
---|
4921 | bool fNewModule = pRecVM == NULL;
|
---|
4922 | if (fNewModule)
|
---|
4923 | {
|
---|
4924 | rc = gmmR0ShModNewPerVM(pGVM, GCPtrModBase, cRegions, paRegions, &pRecVM);
|
---|
4925 | if (RT_SUCCESS(rc))
|
---|
4926 | {
|
---|
4927 | /*
|
---|
4928 | * Find a matching global module, register a new one if needed.
|
---|
4929 | */
|
---|
4930 | PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
|
---|
4931 | pszModuleName, pszVersion, paRegions);
|
---|
4932 | if (!pGblMod)
|
---|
4933 | {
|
---|
4934 | Assert(fNewModule);
|
---|
4935 | rc = gmmR0ShModNewGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
|
---|
4936 | pszModuleName, pszVersion, paRegions, &pGblMod);
|
---|
4937 | if (RT_SUCCESS(rc))
|
---|
4938 | {
|
---|
4939 | pRecVM->pGlobalModule = pGblMod; /* (One referenced returned by gmmR0ShModNewGlobal.) */
|
---|
4940 | Log(("GMMR0RegisterSharedModule: new module %s %s\n", pszModuleName, pszVersion));
|
---|
4941 | }
|
---|
4942 | else
|
---|
4943 | gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/);
|
---|
4944 | }
|
---|
4945 | else
|
---|
4946 | {
|
---|
4947 | Assert(pGblMod->cUsers > 0 && pGblMod->cUsers < UINT32_MAX / 2);
|
---|
4948 | pGblMod->cUsers++;
|
---|
4949 | pRecVM->pGlobalModule = pGblMod;
|
---|
4950 |
|
---|
4951 | Log(("GMMR0RegisterSharedModule: new per vm module %s %s, gbl users %d\n", pszModuleName, pszVersion, pGblMod->cUsers));
|
---|
4952 | }
|
---|
4953 | }
|
---|
4954 | }
|
---|
4955 | else
|
---|
4956 | {
|
---|
4957 | /*
|
---|
4958 | * Attempt to re-register an existing module.
|
---|
4959 | */
|
---|
4960 | PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
|
---|
4961 | pszModuleName, pszVersion, paRegions);
|
---|
4962 | if (pRecVM->pGlobalModule == pGblMod)
|
---|
4963 | {
|
---|
4964 | Log(("GMMR0RegisterSharedModule: already registered %s %s, gbl users %d\n", pszModuleName, pszVersion, pGblMod->cUsers));
|
---|
4965 | rc = VINF_GMM_SHARED_MODULE_ALREADY_REGISTERED;
|
---|
4966 | }
|
---|
4967 | else
|
---|
4968 | {
|
---|
4969 | /** @todo may have to unregister+register when this happens in case it's caused
|
---|
4970 | * by VBoxService crashing and being restarted... */
|
---|
4971 | Log(("GMMR0RegisterSharedModule: Address clash!\n"
|
---|
4972 | " incoming at %RGvLB%#x %s %s rgns %u\n"
|
---|
4973 | " existing at %RGvLB%#x %s %s rgns %u\n",
|
---|
4974 | GCPtrModBase, cbModule, pszModuleName, pszVersion, cRegions,
|
---|
4975 | pRecVM->Core.Key, pRecVM->pGlobalModule->cbModule, pRecVM->pGlobalModule->szName,
|
---|
4976 | pRecVM->pGlobalModule->szVersion, pRecVM->pGlobalModule->cRegions));
|
---|
4977 | rc = VERR_GMM_SHARED_MODULE_ADDRESS_CLASH;
|
---|
4978 | }
|
---|
4979 | }
|
---|
4980 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
4981 | }
|
---|
4982 | else
|
---|
4983 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
4984 |
|
---|
4985 | gmmR0MutexRelease(pGMM);
|
---|
4986 | return rc;
|
---|
4987 | #else
|
---|
4988 |
|
---|
4989 | NOREF(pGVM); NOREF(idCpu); NOREF(enmGuestOS); NOREF(pszModuleName); NOREF(pszVersion);
|
---|
4990 | NOREF(GCPtrModBase); NOREF(cbModule); NOREF(cRegions); NOREF(paRegions);
|
---|
4991 | return VERR_NOT_IMPLEMENTED;
|
---|
4992 | #endif
|
---|
4993 | }
|
---|
4994 |
|
---|
4995 |
|
---|
4996 | /**
|
---|
4997 | * VMMR0 request wrapper for GMMR0RegisterSharedModule.
|
---|
4998 | *
|
---|
4999 | * @returns see GMMR0RegisterSharedModule.
|
---|
5000 | * @param pGVM The global (ring-0) VM structure.
|
---|
5001 | * @param idCpu The VCPU id.
|
---|
5002 | * @param pReq Pointer to the request packet.
|
---|
5003 | */
|
---|
5004 | GMMR0DECL(int) GMMR0RegisterSharedModuleReq(PGVM pGVM, VMCPUID idCpu, PGMMREGISTERSHAREDMODULEREQ pReq)
|
---|
5005 | {
|
---|
5006 | /*
|
---|
5007 | * Validate input and pass it on.
|
---|
5008 | */
|
---|
5009 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
5010 | AssertMsgReturn( pReq->Hdr.cbReq >= sizeof(*pReq)
|
---|
5011 | && pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMREGISTERSHAREDMODULEREQ, aRegions[pReq->cRegions]),
|
---|
5012 | ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
|
---|
5013 |
|
---|
5014 | /* Pass back return code in the request packet to preserve informational codes. (VMMR3CallR0 chokes on them) */
|
---|
5015 | pReq->rc = GMMR0RegisterSharedModule(pGVM, idCpu, pReq->enmGuestOS, pReq->szName, pReq->szVersion,
|
---|
5016 | pReq->GCBaseAddr, pReq->cbModule, pReq->cRegions, pReq->aRegions);
|
---|
5017 | return VINF_SUCCESS;
|
---|
5018 | }
|
---|
5019 |
|
---|
5020 |
|
---|
5021 | /**
|
---|
5022 | * Unregisters a shared module for the VM
|
---|
5023 | *
|
---|
5024 | * @returns VBox status code.
|
---|
5025 | * @param pGVM The global (ring-0) VM structure.
|
---|
5026 | * @param idCpu The VCPU id.
|
---|
5027 | * @param pszModuleName The module name.
|
---|
5028 | * @param pszVersion The module version.
|
---|
5029 | * @param GCPtrModBase The module base address.
|
---|
5030 | * @param cbModule The module size.
|
---|
5031 | */
|
---|
5032 | GMMR0DECL(int) GMMR0UnregisterSharedModule(PGVM pGVM, VMCPUID idCpu, char *pszModuleName, char *pszVersion,
|
---|
5033 | RTGCPTR GCPtrModBase, uint32_t cbModule)
|
---|
5034 | {
|
---|
5035 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
5036 | /*
|
---|
5037 | * Validate input and get the basics.
|
---|
5038 | */
|
---|
5039 | PGMM pGMM;
|
---|
5040 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
5041 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
5042 | if (RT_FAILURE(rc))
|
---|
5043 | return rc;
|
---|
5044 |
|
---|
5045 | AssertPtrReturn(pszModuleName, VERR_INVALID_POINTER);
|
---|
5046 | AssertPtrReturn(pszVersion, VERR_INVALID_POINTER);
|
---|
5047 | if (RT_UNLIKELY(!memchr(pszModuleName, '\0', GMM_SHARED_MODULE_MAX_NAME_STRING)))
|
---|
5048 | return VERR_GMM_MODULE_NAME_TOO_LONG;
|
---|
5049 | if (RT_UNLIKELY(!memchr(pszVersion, '\0', GMM_SHARED_MODULE_MAX_VERSION_STRING)))
|
---|
5050 | return VERR_GMM_MODULE_NAME_TOO_LONG;
|
---|
5051 |
|
---|
5052 | Log(("GMMR0UnregisterSharedModule %s %s base=%RGv size %x\n", pszModuleName, pszVersion, GCPtrModBase, cbModule));
|
---|
5053 |
|
---|
5054 | /*
|
---|
5055 | * Take the semaphore and do some more validations.
|
---|
5056 | */
|
---|
5057 | gmmR0MutexAcquire(pGMM);
|
---|
5058 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
5059 | {
|
---|
5060 | /*
|
---|
5061 | * Locate and remove the specified module.
|
---|
5062 | */
|
---|
5063 | PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)RTAvlGCPtrGet(&pGVM->gmm.s.pSharedModuleTree, GCPtrModBase);
|
---|
5064 | if (pRecVM)
|
---|
5065 | {
|
---|
5066 | /** @todo Do we need to do more validations here, like that the
|
---|
5067 | * name + version + cbModule matches? */
|
---|
5068 | NOREF(cbModule);
|
---|
5069 | Assert(pRecVM->pGlobalModule);
|
---|
5070 | gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/);
|
---|
5071 | }
|
---|
5072 | else
|
---|
5073 | rc = VERR_GMM_SHARED_MODULE_NOT_FOUND;
|
---|
5074 |
|
---|
5075 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
5076 | }
|
---|
5077 | else
|
---|
5078 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
5079 |
|
---|
5080 | gmmR0MutexRelease(pGMM);
|
---|
5081 | return rc;
|
---|
5082 | #else
|
---|
5083 |
|
---|
5084 | NOREF(pGVM); NOREF(idCpu); NOREF(pszModuleName); NOREF(pszVersion); NOREF(GCPtrModBase); NOREF(cbModule);
|
---|
5085 | return VERR_NOT_IMPLEMENTED;
|
---|
5086 | #endif
|
---|
5087 | }
|
---|
5088 |
|
---|
5089 |
|
---|
5090 | /**
|
---|
5091 | * VMMR0 request wrapper for GMMR0UnregisterSharedModule.
|
---|
5092 | *
|
---|
5093 | * @returns see GMMR0UnregisterSharedModule.
|
---|
5094 | * @param pGVM The global (ring-0) VM structure.
|
---|
5095 | * @param idCpu The VCPU id.
|
---|
5096 | * @param pReq Pointer to the request packet.
|
---|
5097 | */
|
---|
5098 | GMMR0DECL(int) GMMR0UnregisterSharedModuleReq(PGVM pGVM, VMCPUID idCpu, PGMMUNREGISTERSHAREDMODULEREQ pReq)
|
---|
5099 | {
|
---|
5100 | /*
|
---|
5101 | * Validate input and pass it on.
|
---|
5102 | */
|
---|
5103 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
5104 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
|
---|
5105 |
|
---|
5106 | return GMMR0UnregisterSharedModule(pGVM, idCpu, pReq->szName, pReq->szVersion, pReq->GCBaseAddr, pReq->cbModule);
|
---|
5107 | }
|
---|
5108 |
|
---|
5109 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
5110 |
|
---|
5111 | /**
|
---|
5112 | * Increase the use count of a shared page, the page is known to exist and be valid and such.
|
---|
5113 | *
|
---|
5114 | * @param pGMM Pointer to the GMM instance.
|
---|
5115 | * @param pGVM Pointer to the GVM instance.
|
---|
5116 | * @param pPage The page structure.
|
---|
5117 | */
|
---|
5118 | DECLINLINE(void) gmmR0UseSharedPage(PGMM pGMM, PGVM pGVM, PGMMPAGE pPage)
|
---|
5119 | {
|
---|
5120 | Assert(pGMM->cSharedPages > 0);
|
---|
5121 | Assert(pGMM->cAllocatedPages > 0);
|
---|
5122 |
|
---|
5123 | pGMM->cDuplicatePages++;
|
---|
5124 |
|
---|
5125 | pPage->Shared.cRefs++;
|
---|
5126 | pGVM->gmm.s.Stats.cSharedPages++;
|
---|
5127 | pGVM->gmm.s.Stats.Allocated.cBasePages++;
|
---|
5128 | }
|
---|
5129 |
|
---|
5130 |
|
---|
5131 | /**
|
---|
5132 | * Converts a private page to a shared page, the page is known to exist and be valid and such.
|
---|
5133 | *
|
---|
5134 | * @param pGMM Pointer to the GMM instance.
|
---|
5135 | * @param pGVM Pointer to the GVM instance.
|
---|
5136 | * @param HCPhys Host physical address
|
---|
5137 | * @param idPage The Page ID
|
---|
5138 | * @param pPage The page structure.
|
---|
5139 | * @param pPageDesc Shared page descriptor
|
---|
5140 | */
|
---|
5141 | DECLINLINE(void) gmmR0ConvertToSharedPage(PGMM pGMM, PGVM pGVM, RTHCPHYS HCPhys, uint32_t idPage, PGMMPAGE pPage,
|
---|
5142 | PGMMSHAREDPAGEDESC pPageDesc)
|
---|
5143 | {
|
---|
5144 | PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
|
---|
5145 | Assert(pChunk);
|
---|
5146 | Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
|
---|
5147 | Assert(GMM_PAGE_IS_PRIVATE(pPage));
|
---|
5148 |
|
---|
5149 | pChunk->cPrivate--;
|
---|
5150 | pChunk->cShared++;
|
---|
5151 |
|
---|
5152 | pGMM->cSharedPages++;
|
---|
5153 |
|
---|
5154 | pGVM->gmm.s.Stats.cSharedPages++;
|
---|
5155 | pGVM->gmm.s.Stats.cPrivatePages--;
|
---|
5156 |
|
---|
5157 | /* Modify the page structure. */
|
---|
5158 | pPage->Shared.pfn = (uint32_t)(uint64_t)(HCPhys >> GUEST_PAGE_SHIFT);
|
---|
5159 | pPage->Shared.cRefs = 1;
|
---|
5160 | #ifdef VBOX_STRICT
|
---|
5161 | pPageDesc->u32StrictChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage);
|
---|
5162 | pPage->Shared.u14Checksum = pPageDesc->u32StrictChecksum;
|
---|
5163 | #else
|
---|
5164 | NOREF(pPageDesc);
|
---|
5165 | pPage->Shared.u14Checksum = 0;
|
---|
5166 | #endif
|
---|
5167 | pPage->Shared.u2State = GMM_PAGE_STATE_SHARED;
|
---|
5168 | }
|
---|
5169 |
|
---|
5170 |
|
---|
5171 | static int gmmR0SharedModuleCheckPageFirstTime(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULE pModule,
|
---|
5172 | unsigned idxRegion, unsigned idxPage,
|
---|
5173 | PGMMSHAREDPAGEDESC pPageDesc, PGMMSHAREDREGIONDESC pGlobalRegion)
|
---|
5174 | {
|
---|
5175 | NOREF(pModule);
|
---|
5176 |
|
---|
5177 | /* Easy case: just change the internal page type. */
|
---|
5178 | PGMMPAGE pPage = gmmR0GetPage(pGMM, pPageDesc->idPage);
|
---|
5179 | AssertMsgReturn(pPage, ("idPage=%#x (GCPhys=%RGp HCPhys=%RHp idxRegion=%#x idxPage=%#x) #1\n",
|
---|
5180 | pPageDesc->idPage, pPageDesc->GCPhys, pPageDesc->HCPhys, idxRegion, idxPage),
|
---|
5181 | VERR_PGM_PHYS_INVALID_PAGE_ID);
|
---|
5182 | NOREF(idxRegion);
|
---|
5183 |
|
---|
5184 | AssertMsg(pPageDesc->GCPhys == (pPage->Private.pfn << 12), ("desc %RGp gmm %RGp\n", pPageDesc->HCPhys, (pPage->Private.pfn << 12)));
|
---|
5185 |
|
---|
5186 | gmmR0ConvertToSharedPage(pGMM, pGVM, pPageDesc->HCPhys, pPageDesc->idPage, pPage, pPageDesc);
|
---|
5187 |
|
---|
5188 | /* Keep track of these references. */
|
---|
5189 | pGlobalRegion->paidPages[idxPage] = pPageDesc->idPage;
|
---|
5190 |
|
---|
5191 | return VINF_SUCCESS;
|
---|
5192 | }
|
---|
5193 |
|
---|
5194 | /**
|
---|
5195 | * Checks specified shared module range for changes
|
---|
5196 | *
|
---|
5197 | * Performs the following tasks:
|
---|
5198 | * - If a shared page is new, then it changes the GMM page type to shared and
|
---|
5199 | * returns it in the pPageDesc descriptor.
|
---|
5200 | * - If a shared page already exists, then it checks if the VM page is
|
---|
5201 | * identical and if so frees the VM page and returns the shared page in
|
---|
5202 | * pPageDesc descriptor.
|
---|
5203 | *
|
---|
5204 | * @remarks ASSUMES the caller has acquired the GMM semaphore!!
|
---|
5205 | *
|
---|
5206 | * @returns VBox status code.
|
---|
5207 | * @param pGVM Pointer to the GVM instance data.
|
---|
5208 | * @param pModule Module description
|
---|
5209 | * @param idxRegion Region index
|
---|
5210 | * @param idxPage Page index
|
---|
5211 | * @param pPageDesc Page descriptor
|
---|
5212 | */
|
---|
5213 | GMMR0DECL(int) GMMR0SharedModuleCheckPage(PGVM pGVM, PGMMSHAREDMODULE pModule, uint32_t idxRegion, uint32_t idxPage,
|
---|
5214 | PGMMSHAREDPAGEDESC pPageDesc)
|
---|
5215 | {
|
---|
5216 | int rc;
|
---|
5217 | PGMM pGMM;
|
---|
5218 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
5219 | pPageDesc->u32StrictChecksum = 0;
|
---|
5220 |
|
---|
5221 | AssertMsgReturn(idxRegion < pModule->cRegions,
|
---|
5222 | ("idxRegion=%#x cRegions=%#x %s %s\n", idxRegion, pModule->cRegions, pModule->szName, pModule->szVersion),
|
---|
5223 | VERR_INVALID_PARAMETER);
|
---|
5224 |
|
---|
5225 | uint32_t const cPages = pModule->aRegions[idxRegion].cb >> GUEST_PAGE_SHIFT;
|
---|
5226 | AssertMsgReturn(idxPage < cPages,
|
---|
5227 | ("idxRegion=%#x cRegions=%#x %s %s\n", idxRegion, pModule->cRegions, pModule->szName, pModule->szVersion),
|
---|
5228 | VERR_INVALID_PARAMETER);
|
---|
5229 |
|
---|
5230 | LogFlow(("GMMR0SharedModuleCheckRange %s base %RGv region %d idxPage %d\n", pModule->szName, pModule->Core.Key, idxRegion, idxPage));
|
---|
5231 |
|
---|
5232 | /*
|
---|
5233 | * First time; create a page descriptor array.
|
---|
5234 | */
|
---|
5235 | PGMMSHAREDREGIONDESC pGlobalRegion = &pModule->aRegions[idxRegion];
|
---|
5236 | if (!pGlobalRegion->paidPages)
|
---|
5237 | {
|
---|
5238 | Log(("Allocate page descriptor array for %d pages\n", cPages));
|
---|
5239 | pGlobalRegion->paidPages = (uint32_t *)RTMemAlloc(cPages * sizeof(pGlobalRegion->paidPages[0]));
|
---|
5240 | AssertReturn(pGlobalRegion->paidPages, VERR_NO_MEMORY);
|
---|
5241 |
|
---|
5242 | /* Invalidate all descriptors. */
|
---|
5243 | uint32_t i = cPages;
|
---|
5244 | while (i-- > 0)
|
---|
5245 | pGlobalRegion->paidPages[i] = NIL_GMM_PAGEID;
|
---|
5246 | }
|
---|
5247 |
|
---|
5248 | /*
|
---|
5249 | * We've seen this shared page for the first time?
|
---|
5250 | */
|
---|
5251 | if (pGlobalRegion->paidPages[idxPage] == NIL_GMM_PAGEID)
|
---|
5252 | {
|
---|
5253 | Log(("New shared page guest %RGp host %RHp\n", pPageDesc->GCPhys, pPageDesc->HCPhys));
|
---|
5254 | return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion);
|
---|
5255 | }
|
---|
5256 |
|
---|
5257 | /*
|
---|
5258 | * We've seen it before...
|
---|
5259 | */
|
---|
5260 | Log(("Replace existing page guest %RGp host %RHp id %#x -> id %#x\n",
|
---|
5261 | pPageDesc->GCPhys, pPageDesc->HCPhys, pPageDesc->idPage, pGlobalRegion->paidPages[idxPage]));
|
---|
5262 | Assert(pPageDesc->idPage != pGlobalRegion->paidPages[idxPage]);
|
---|
5263 |
|
---|
5264 | /*
|
---|
5265 | * Get the shared page source.
|
---|
5266 | */
|
---|
5267 | PGMMPAGE pPage = gmmR0GetPage(pGMM, pGlobalRegion->paidPages[idxPage]);
|
---|
5268 | AssertMsgReturn(pPage, ("idPage=%#x (idxRegion=%#x idxPage=%#x) #2\n", pPageDesc->idPage, idxRegion, idxPage),
|
---|
5269 | VERR_PGM_PHYS_INVALID_PAGE_ID);
|
---|
5270 |
|
---|
5271 | if (pPage->Common.u2State != GMM_PAGE_STATE_SHARED)
|
---|
5272 | {
|
---|
5273 | /*
|
---|
5274 | * Page was freed at some point; invalidate this entry.
|
---|
5275 | */
|
---|
5276 | /** @todo this isn't really bullet proof. */
|
---|
5277 | Log(("Old shared page was freed -> create a new one\n"));
|
---|
5278 | pGlobalRegion->paidPages[idxPage] = NIL_GMM_PAGEID;
|
---|
5279 | return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion);
|
---|
5280 | }
|
---|
5281 |
|
---|
5282 | Log(("Replace existing page guest host %RHp -> %RHp\n", pPageDesc->HCPhys, ((uint64_t)pPage->Shared.pfn) << GUEST_PAGE_SHIFT));
|
---|
5283 |
|
---|
5284 | /*
|
---|
5285 | * Calculate the virtual address of the local page.
|
---|
5286 | */
|
---|
5287 | PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pPageDesc->idPage >> GMM_CHUNKID_SHIFT);
|
---|
5288 | AssertMsgReturn(pChunk, ("idPage=%#x (idxRegion=%#x idxPage=%#x) #4\n", pPageDesc->idPage, idxRegion, idxPage),
|
---|
5289 | VERR_PGM_PHYS_INVALID_PAGE_ID);
|
---|
5290 |
|
---|
5291 | uint8_t *pbChunk;
|
---|
5292 | AssertMsgReturn(gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk),
|
---|
5293 | ("idPage=%#x (idxRegion=%#x idxPage=%#x) #3\n", pPageDesc->idPage, idxRegion, idxPage),
|
---|
5294 | VERR_PGM_PHYS_INVALID_PAGE_ID);
|
---|
5295 | uint8_t *pbLocalPage = pbChunk + ((pPageDesc->idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
|
---|
5296 |
|
---|
5297 | /*
|
---|
5298 | * Calculate the virtual address of the shared page.
|
---|
5299 | */
|
---|
5300 | pChunk = gmmR0GetChunk(pGMM, pGlobalRegion->paidPages[idxPage] >> GMM_CHUNKID_SHIFT);
|
---|
5301 | Assert(pChunk); /* can't fail as gmmR0GetPage succeeded. */
|
---|
5302 |
|
---|
5303 | /*
|
---|
5304 | * Get the virtual address of the physical page; map the chunk into the VM
|
---|
5305 | * process if not already done.
|
---|
5306 | */
|
---|
5307 | if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
|
---|
5308 | {
|
---|
5309 | Log(("Map chunk into process!\n"));
|
---|
5310 | rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk);
|
---|
5311 | AssertRCReturn(rc, rc);
|
---|
5312 | }
|
---|
5313 | uint8_t *pbSharedPage = pbChunk + ((pGlobalRegion->paidPages[idxPage] & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
|
---|
5314 |
|
---|
5315 | #ifdef VBOX_STRICT
|
---|
5316 | pPageDesc->u32StrictChecksum = RTCrc32(pbSharedPage, GUEST_PAGE_SIZE);
|
---|
5317 | uint32_t uChecksum = pPageDesc->u32StrictChecksum & UINT32_C(0x00003fff);
|
---|
5318 | AssertMsg(!uChecksum || uChecksum == pPage->Shared.u14Checksum || !pPage->Shared.u14Checksum,
|
---|
5319 | ("%#x vs %#x - idPage=%#x - %s %s\n", uChecksum, pPage->Shared.u14Checksum,
|
---|
5320 | pGlobalRegion->paidPages[idxPage], pModule->szName, pModule->szVersion));
|
---|
5321 | #endif
|
---|
5322 |
|
---|
5323 | if (memcmp(pbSharedPage, pbLocalPage, GUEST_PAGE_SIZE))
|
---|
5324 | {
|
---|
5325 | Log(("Unexpected differences found between local and shared page; skip\n"));
|
---|
5326 | /* Signal to the caller that this one hasn't changed. */
|
---|
5327 | pPageDesc->idPage = NIL_GMM_PAGEID;
|
---|
5328 | return VINF_SUCCESS;
|
---|
5329 | }
|
---|
5330 |
|
---|
5331 | /*
|
---|
5332 | * Free the old local page.
|
---|
5333 | */
|
---|
5334 | GMMFREEPAGEDESC PageDesc;
|
---|
5335 | PageDesc.idPage = pPageDesc->idPage;
|
---|
5336 | rc = gmmR0FreePages(pGMM, pGVM, 1, &PageDesc, GMMACCOUNT_BASE);
|
---|
5337 | AssertRCReturn(rc, rc);
|
---|
5338 |
|
---|
5339 | gmmR0UseSharedPage(pGMM, pGVM, pPage);
|
---|
5340 |
|
---|
5341 | /*
|
---|
5342 | * Pass along the new physical address & page id.
|
---|
5343 | */
|
---|
5344 | pPageDesc->HCPhys = ((uint64_t)pPage->Shared.pfn) << GUEST_PAGE_SHIFT;
|
---|
5345 | pPageDesc->idPage = pGlobalRegion->paidPages[idxPage];
|
---|
5346 |
|
---|
5347 | return VINF_SUCCESS;
|
---|
5348 | }
|
---|
5349 |
|
---|
5350 |
|
---|
5351 | /**
|
---|
5352 | * RTAvlGCPtrDestroy callback.
|
---|
5353 | *
|
---|
5354 | * @returns 0 or VERR_GMM_INSTANCE.
|
---|
5355 | * @param pNode The node to destroy.
|
---|
5356 | * @param pvArgs Pointer to an argument packet.
|
---|
5357 | */
|
---|
5358 | static DECLCALLBACK(int) gmmR0CleanupSharedModule(PAVLGCPTRNODECORE pNode, void *pvArgs)
|
---|
5359 | {
|
---|
5360 | gmmR0ShModDeletePerVM(((GMMR0SHMODPERVMDTORARGS *)pvArgs)->pGMM,
|
---|
5361 | ((GMMR0SHMODPERVMDTORARGS *)pvArgs)->pGVM,
|
---|
5362 | (PGMMSHAREDMODULEPERVM)pNode,
|
---|
5363 | false /*fRemove*/);
|
---|
5364 | return VINF_SUCCESS;
|
---|
5365 | }
|
---|
5366 |
|
---|
5367 |
|
---|
5368 | /**
|
---|
5369 | * Used by GMMR0CleanupVM to clean up shared modules.
|
---|
5370 | *
|
---|
5371 | * This is called without taking the GMM lock so that it can be yielded as
|
---|
5372 | * needed here.
|
---|
5373 | *
|
---|
5374 | * @param pGMM The GMM handle.
|
---|
5375 | * @param pGVM The global VM handle.
|
---|
5376 | */
|
---|
5377 | static void gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM)
|
---|
5378 | {
|
---|
5379 | gmmR0MutexAcquire(pGMM);
|
---|
5380 | GMM_CHECK_SANITY_UPON_ENTERING(pGMM);
|
---|
5381 |
|
---|
5382 | GMMR0SHMODPERVMDTORARGS Args;
|
---|
5383 | Args.pGVM = pGVM;
|
---|
5384 | Args.pGMM = pGMM;
|
---|
5385 | RTAvlGCPtrDestroy(&pGVM->gmm.s.pSharedModuleTree, gmmR0CleanupSharedModule, &Args);
|
---|
5386 |
|
---|
5387 | AssertMsg(pGVM->gmm.s.Stats.cShareableModules == 0, ("%d\n", pGVM->gmm.s.Stats.cShareableModules));
|
---|
5388 | pGVM->gmm.s.Stats.cShareableModules = 0;
|
---|
5389 |
|
---|
5390 | gmmR0MutexRelease(pGMM);
|
---|
5391 | }
|
---|
5392 |
|
---|
5393 | #endif /* VBOX_WITH_PAGE_SHARING */
|
---|
5394 |
|
---|
5395 | /**
|
---|
5396 | * Removes all shared modules for the specified VM
|
---|
5397 | *
|
---|
5398 | * @returns VBox status code.
|
---|
5399 | * @param pGVM The global (ring-0) VM structure.
|
---|
5400 | * @param idCpu The VCPU id.
|
---|
5401 | */
|
---|
5402 | GMMR0DECL(int) GMMR0ResetSharedModules(PGVM pGVM, VMCPUID idCpu)
|
---|
5403 | {
|
---|
5404 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
5405 | /*
|
---|
5406 | * Validate input and get the basics.
|
---|
5407 | */
|
---|
5408 | PGMM pGMM;
|
---|
5409 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
5410 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
5411 | if (RT_FAILURE(rc))
|
---|
5412 | return rc;
|
---|
5413 |
|
---|
5414 | /*
|
---|
5415 | * Take the semaphore and do some more validations.
|
---|
5416 | */
|
---|
5417 | gmmR0MutexAcquire(pGMM);
|
---|
5418 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
5419 | {
|
---|
5420 | Log(("GMMR0ResetSharedModules\n"));
|
---|
5421 | GMMR0SHMODPERVMDTORARGS Args;
|
---|
5422 | Args.pGVM = pGVM;
|
---|
5423 | Args.pGMM = pGMM;
|
---|
5424 | RTAvlGCPtrDestroy(&pGVM->gmm.s.pSharedModuleTree, gmmR0CleanupSharedModule, &Args);
|
---|
5425 | pGVM->gmm.s.Stats.cShareableModules = 0;
|
---|
5426 |
|
---|
5427 | rc = VINF_SUCCESS;
|
---|
5428 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
5429 | }
|
---|
5430 | else
|
---|
5431 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
5432 |
|
---|
5433 | gmmR0MutexRelease(pGMM);
|
---|
5434 | return rc;
|
---|
5435 | #else
|
---|
5436 | RT_NOREF(pGVM, idCpu);
|
---|
5437 | return VERR_NOT_IMPLEMENTED;
|
---|
5438 | #endif
|
---|
5439 | }
|
---|
5440 |
|
---|
5441 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
5442 |
|
---|
5443 | /**
|
---|
5444 | * Tree enumeration callback for checking a shared module.
|
---|
5445 | */
|
---|
5446 | static DECLCALLBACK(int) gmmR0CheckSharedModule(PAVLGCPTRNODECORE pNode, void *pvUser)
|
---|
5447 | {
|
---|
5448 | GMMCHECKSHAREDMODULEINFO *pArgs = (GMMCHECKSHAREDMODULEINFO*)pvUser;
|
---|
5449 | PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)pNode;
|
---|
5450 | PGMMSHAREDMODULE pGblMod = pRecVM->pGlobalModule;
|
---|
5451 |
|
---|
5452 | Log(("gmmR0CheckSharedModule: check %s %s base=%RGv size=%x\n",
|
---|
5453 | pGblMod->szName, pGblMod->szVersion, pGblMod->Core.Key, pGblMod->cbModule));
|
---|
5454 |
|
---|
5455 | int rc = PGMR0SharedModuleCheck(pArgs->pGVM, pArgs->pGVM, pArgs->idCpu, pGblMod, pRecVM->aRegionsGCPtrs);
|
---|
5456 | if (RT_FAILURE(rc))
|
---|
5457 | return rc;
|
---|
5458 | return VINF_SUCCESS;
|
---|
5459 | }
|
---|
5460 |
|
---|
5461 | #endif /* VBOX_WITH_PAGE_SHARING */
|
---|
5462 |
|
---|
5463 | /**
|
---|
5464 | * Check all shared modules for the specified VM.
|
---|
5465 | *
|
---|
5466 | * @returns VBox status code.
|
---|
5467 | * @param pGVM The global (ring-0) VM structure.
|
---|
5468 | * @param idCpu The calling EMT number.
|
---|
5469 | * @thread EMT(idCpu)
|
---|
5470 | */
|
---|
5471 | GMMR0DECL(int) GMMR0CheckSharedModules(PGVM pGVM, VMCPUID idCpu)
|
---|
5472 | {
|
---|
5473 | #ifdef VBOX_WITH_PAGE_SHARING
|
---|
5474 | /*
|
---|
5475 | * Validate input and get the basics.
|
---|
5476 | */
|
---|
5477 | PGMM pGMM;
|
---|
5478 | GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
|
---|
5479 | int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
|
---|
5480 | if (RT_FAILURE(rc))
|
---|
5481 | return rc;
|
---|
5482 |
|
---|
5483 | # ifndef DEBUG_sandervl
|
---|
5484 | /*
|
---|
5485 | * Take the semaphore and do some more validations.
|
---|
5486 | */
|
---|
5487 | gmmR0MutexAcquire(pGMM);
|
---|
5488 | # endif
|
---|
5489 | if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
|
---|
5490 | {
|
---|
5491 | /*
|
---|
5492 | * Walk the tree, checking each module.
|
---|
5493 | */
|
---|
5494 | Log(("GMMR0CheckSharedModules\n"));
|
---|
5495 |
|
---|
5496 | GMMCHECKSHAREDMODULEINFO Args;
|
---|
5497 | Args.pGVM = pGVM;
|
---|
5498 | Args.idCpu = idCpu;
|
---|
5499 | rc = RTAvlGCPtrDoWithAll(&pGVM->gmm.s.pSharedModuleTree, true /* fFromLeft */, gmmR0CheckSharedModule, &Args);
|
---|
5500 |
|
---|
5501 | Log(("GMMR0CheckSharedModules done (rc=%Rrc)!\n", rc));
|
---|
5502 | GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
|
---|
5503 | }
|
---|
5504 | else
|
---|
5505 | rc = VERR_GMM_IS_NOT_SANE;
|
---|
5506 |
|
---|
5507 | # ifndef DEBUG_sandervl
|
---|
5508 | gmmR0MutexRelease(pGMM);
|
---|
5509 | # endif
|
---|
5510 | return rc;
|
---|
5511 | #else
|
---|
5512 | RT_NOREF(pGVM, idCpu);
|
---|
5513 | return VERR_NOT_IMPLEMENTED;
|
---|
5514 | #endif
|
---|
5515 | }
|
---|
5516 |
|
---|
5517 | #ifdef VBOX_STRICT
|
---|
5518 |
|
---|
5519 | /**
|
---|
5520 | * Worker for GMMR0FindDuplicatePageReq.
|
---|
5521 | *
|
---|
5522 | * @returns true if duplicate, false if not.
|
---|
5523 | */
|
---|
5524 | static bool gmmR0FindDupPageInChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, uint8_t const *pbSourcePage)
|
---|
5525 | {
|
---|
5526 | bool fFoundDuplicate = false;
|
---|
5527 | /* Only take chunks not mapped into this VM process; not entirely correct. */
|
---|
5528 | uint8_t *pbChunk;
|
---|
5529 | if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
|
---|
5530 | {
|
---|
5531 | int rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk);
|
---|
5532 | if (RT_SUCCESS(rc))
|
---|
5533 | {
|
---|
5534 | /*
|
---|
5535 | * Look for duplicate pages
|
---|
5536 | */
|
---|
5537 | uintptr_t iPage = GMM_CHUNK_NUM_PAGES;
|
---|
5538 | while (iPage-- > 0)
|
---|
5539 | {
|
---|
5540 | if (GMM_PAGE_IS_PRIVATE(&pChunk->aPages[iPage]))
|
---|
5541 | {
|
---|
5542 | uint8_t *pbDestPage = pbChunk + (iPage << GUEST_PAGE_SHIFT);
|
---|
5543 | if (!memcmp(pbSourcePage, pbDestPage, GUEST_PAGE_SIZE))
|
---|
5544 | {
|
---|
5545 | fFoundDuplicate = true;
|
---|
5546 | break;
|
---|
5547 | }
|
---|
5548 | }
|
---|
5549 | }
|
---|
5550 | gmmR0UnmapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/);
|
---|
5551 | }
|
---|
5552 | }
|
---|
5553 | return fFoundDuplicate;
|
---|
5554 | }
|
---|
5555 |
|
---|
5556 |
|
---|
5557 | /**
|
---|
5558 | * Find a duplicate of the specified page in other active VMs
|
---|
5559 | *
|
---|
5560 | * @returns VBox status code.
|
---|
5561 | * @param pGVM The global (ring-0) VM structure.
|
---|
5562 | * @param pReq Pointer to the request packet.
|
---|
5563 | */
|
---|
5564 | GMMR0DECL(int) GMMR0FindDuplicatePageReq(PGVM pGVM, |
---|