VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/GMMR0.cpp

Last change on this file was 106061, checked in by vboxsync, 3 months ago

Copyright year updates by scm.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 204.3 KB
Line 
1/* $Id: GMMR0.cpp 106061 2024-09-16 14:03:52Z vboxsync $ */
2/** @file
3 * GMM - Global Memory Manager.
4 */
5
6/*
7 * Copyright (C) 2007-2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/** @page pg_gmm GMM - The Global Memory Manager
30 *
31 * As the name indicates, this component is responsible for global memory
32 * management. Currently only guest RAM is allocated from the GMM, but this
33 * may change to include shadow page tables and other bits later.
34 *
35 * Guest RAM is managed as individual pages, but allocated from the host OS
36 * in chunks for reasons of portability / efficiency. To minimize the memory
37 * footprint all tracking structure must be as small as possible without
38 * unnecessary performance penalties.
39 *
40 * The allocation chunks has fixed sized, the size defined at compile time
41 * by the #GMM_CHUNK_SIZE \#define.
42 *
43 * Each chunk is given an unique ID. Each page also has a unique ID. The
44 * relationship between the two IDs is:
45 * @code
46 * GMM_CHUNK_SHIFT = log2(GMM_CHUNK_SIZE / GUEST_PAGE_SIZE);
47 * idPage = (idChunk << GMM_CHUNK_SHIFT) | iPage;
48 * @endcode
49 * Where iPage is the index of the page within the chunk. This ID scheme
50 * permits for efficient chunk and page lookup, but it relies on the chunk size
51 * to be set at compile time. The chunks are organized in an AVL tree with their
52 * IDs being the keys.
53 *
54 * @todo Scope the chunk+page IDs based on config setting: per VM,
55 * per user (default), or global. This will prevent ring-3 code screwing
56 * around with random page IDs from accessing someone else's data in the
57 * default config. This would let us move HCPhys out of PGMPAGE when
58 * restricting it to ring-0 only, w/o requiring any additional ring-0 per
59 * page data (prereq mmio2 must go via GMM). See @bugref{10696} for more.
60 *
61 * The physical address of each page in an allocation chunk is maintained by
62 * the #RTR0MEMOBJ and obtained using #RTR0MemObjGetPagePhysAddr. There is no
63 * need to duplicate this information (it'll cost 8-bytes per page if we did).
64 *
65 * So what do we need to track per page? Most importantly we need to know
66 * which state the page is in:
67 * - Private - Allocated for (eventually) backing one particular VM page.
68 * - Shared - Readonly page that is used by one or more VMs and treated
69 * as COW by PGM.
70 * - Free - Not used by anyone.
71 *
72 * For the page replacement operations (sharing, defragmenting and freeing)
73 * to be somewhat efficient, private pages needs to be associated with a
74 * particular page in a particular VM.
75 *
76 * Tracking the usage of shared pages is impractical and expensive, so we'll
77 * settle for a reference counting system instead.
78 *
79 * Free pages will be chained on LIFOs
80 *
81 * On 64-bit systems we will use a 64-bit bitfield per page, while on 32-bit
82 * systems a 32-bit bitfield will have to suffice because of address space
83 * limitations. The #GMMPAGE structure shows the details.
84 *
85 *
86 * @section sec_gmm_alloc_strat Page Allocation Strategy
87 *
88 * The strategy for allocating pages has to take fragmentation and shared
89 * pages into account, or we may end up with with 2000 chunks with only
90 * a few pages in each. Shared pages cannot easily be reallocated because
91 * of the inaccurate usage accounting (see above). Private pages can be
92 * reallocated by a defragmentation thread in the same manner that sharing
93 * is done.
94 *
95 * The first approach is to manage the free pages in two sets depending on
96 * whether they are mainly for the allocation of shared or private pages.
97 * In the initial implementation there will be almost no possibility for
98 * mixing shared and private pages in the same chunk (only if we're really
99 * stressed on memory), but when we implement forking of VMs and have to
100 * deal with lots of COW pages it'll start getting kind of interesting.
101 *
102 * The sets are lists of chunks with approximately the same number of
103 * free pages. Say the chunk size is 1MB, meaning 256 pages, and a set
104 * consists of 16 lists. So, the first list will contain the chunks with
105 * 1-7 free pages, the second covers 8-15, and so on. The chunks will be
106 * moved between the lists as pages are freed up or allocated.
107 *
108 *
109 * @section sec_gmm_costs Costs
110 *
111 * The per page cost in kernel space is 32-bit plus whatever RTR0MEMOBJ
112 * entails. In addition there is the chunk cost of approximately
113 * (sizeof(RT0MEMOBJ) + sizeof(CHUNK)) / 2^CHUNK_SHIFT bytes per page.
114 *
115 * On Windows the per page #RTR0MEMOBJ cost is 32-bit on 32-bit windows
116 * and 64-bit on 64-bit windows (a PFN_NUMBER in the MDL). So, 64-bit per page.
117 * The cost on Linux is identical, but here it's because of sizeof(struct page *).
118 *
119 *
120 * @section sec_gmm_legacy Legacy Mode for Non-Tier-1 Platforms
121 *
122 * In legacy mode the page source is locked user pages and not
123 * #RTR0MemObjAllocPhysNC, this means that a page can only be allocated
124 * by the VM that locked it. We will make no attempt at implementing
125 * page sharing on these systems, just do enough to make it all work.
126 *
127 * @note With 6.1 really dropping 32-bit support, the legacy mode is obsoleted
128 * under the assumption that there is sufficient kernel virtual address
129 * space to map all of the guest memory allocations. So, we'll be using
130 * #RTR0MemObjAllocPage on some platforms as an alternative to
131 * #RTR0MemObjAllocPhysNC.
132 *
133 *
134 * @subsection sub_gmm_locking Serializing
135 *
136 * One simple fast mutex will be employed in the initial implementation, not
137 * two as mentioned in @ref sec_pgmPhys_Serializing.
138 *
139 * @see @ref sec_pgmPhys_Serializing
140 *
141 *
142 * @section sec_gmm_overcommit Memory Over-Commitment Management
143 *
144 * The GVM will have to do the system wide memory over-commitment
145 * management. My current ideas are:
146 * - Per VM oc policy that indicates how much to initially commit
147 * to it and what to do in a out-of-memory situation.
148 * - Prevent overtaxing the host.
149 *
150 * There are some challenges here, the main ones are configurability and
151 * security. Should we for instance permit anyone to request 100% memory
152 * commitment? Who should be allowed to do runtime adjustments of the
153 * config. And how to prevent these settings from being lost when the last
154 * VM process exits? The solution is probably to have an optional root
155 * daemon the will keep VMMR0.r0 in memory and enable the security measures.
156 *
157 *
158 *
159 * @section sec_gmm_numa NUMA
160 *
161 * NUMA considerations will be designed and implemented a bit later.
162 *
163 * The preliminary guesses is that we will have to try allocate memory as
164 * close as possible to the CPUs the VM is executed on (EMT and additional CPU
165 * threads). Which means it's mostly about allocation and sharing policies.
166 * Both the scheduler and allocator interface will to supply some NUMA info
167 * and we'll need to have a way to calc access costs.
168 *
169 */
170
171
172/*********************************************************************************************************************************
173* Header Files *
174*********************************************************************************************************************************/
175#define LOG_GROUP LOG_GROUP_GMM
176#include <VBox/rawpci.h>
177#include <VBox/vmm/gmm.h>
178#include "GMMR0Internal.h"
179#include <VBox/vmm/vmcc.h>
180#include <VBox/vmm/pgm.h>
181#include <VBox/log.h>
182#include <VBox/param.h>
183#include <VBox/err.h>
184#include <VBox/VMMDev.h>
185#include <iprt/asm.h>
186#include <iprt/avl.h>
187#ifdef VBOX_STRICT
188# include <iprt/crc.h>
189#endif
190#include <iprt/critsect.h>
191#include <iprt/list.h>
192#include <iprt/mem.h>
193#include <iprt/memobj.h>
194#include <iprt/mp.h>
195#include <iprt/semaphore.h>
196#include <iprt/spinlock.h>
197#include <iprt/string.h>
198#include <iprt/time.h>
199
200/* This is 64-bit only code now. */
201#if HC_ARCH_BITS != 64 || ARCH_BITS != 64
202# error "This is 64-bit only code"
203#endif
204
205
206/*********************************************************************************************************************************
207* Defined Constants And Macros *
208*********************************************************************************************************************************/
209/** @def VBOX_USE_CRIT_SECT_FOR_GIANT
210 * Use a critical section instead of a fast mutex for the giant GMM lock.
211 *
212 * @remarks This is primarily a way of avoiding the deadlock checks in the
213 * windows driver verifier. */
214#if defined(RT_OS_WINDOWS) || defined(RT_OS_DARWIN) || defined(DOXYGEN_RUNNING)
215# define VBOX_USE_CRIT_SECT_FOR_GIANT
216#endif
217
218
219/*********************************************************************************************************************************
220* Structures and Typedefs *
221*********************************************************************************************************************************/
222/** Pointer to set of free chunks. */
223typedef struct GMMCHUNKFREESET *PGMMCHUNKFREESET;
224
225/**
226 * The per-page tracking structure employed by the GMM.
227 *
228 * Because of the different layout on 32-bit and 64-bit hosts in earlier
229 * versions of the code, macros are used to get and set some of the data.
230 */
231typedef union GMMPAGE
232{
233 /** Unsigned integer view. */
234 uint64_t u;
235
236 /** The common view. */
237 struct GMMPAGECOMMON
238 {
239 uint32_t uStuff1 : 32;
240 uint32_t uStuff2 : 30;
241 /** The page state. */
242 uint32_t u2State : 2;
243 } Common;
244
245 /** The view of a private page. */
246 struct GMMPAGEPRIVATE
247 {
248 /** The guest page frame number. (Max addressable: 2 ^ 44 - 16) */
249 uint32_t pfn;
250 /** The GVM handle. (64K VMs) */
251 uint32_t hGVM : 16;
252 /** Reserved. */
253 uint32_t u16Reserved : 14;
254 /** The page state. */
255 uint32_t u2State : 2;
256 } Private;
257
258 /** The view of a shared page. */
259 struct GMMPAGESHARED
260 {
261 /** The host page frame number. (Max addressable: 2 ^ 44 - 16) */
262 uint32_t pfn;
263 /** The reference count (64K VMs). */
264 uint32_t cRefs : 16;
265 /** Used for debug checksumming. */
266 uint32_t u14Checksum : 14;
267 /** The page state. */
268 uint32_t u2State : 2;
269 } Shared;
270
271 /** The view of a free page. */
272 struct GMMPAGEFREE
273 {
274 /** The index of the next page in the free list. UINT16_MAX is NIL. */
275 uint16_t iNext;
276 /** Reserved. Checksum or something? */
277 uint16_t u16Reserved0;
278 /** Reserved. Checksum or something? */
279 uint32_t u30Reserved1 : 29;
280 /** Set if the page was zeroed. */
281 uint32_t fZeroed : 1;
282 /** The page state. */
283 uint32_t u2State : 2;
284 } Free;
285} GMMPAGE;
286AssertCompileSize(GMMPAGE, sizeof(RTHCUINTPTR));
287/** Pointer to a GMMPAGE. */
288typedef GMMPAGE *PGMMPAGE;
289
290
291/** @name The Page States.
292 * @{ */
293/** A private page. */
294#define GMM_PAGE_STATE_PRIVATE 0
295/** A shared page. */
296#define GMM_PAGE_STATE_SHARED 2
297/** A free page. */
298#define GMM_PAGE_STATE_FREE 3
299/** @} */
300
301
302/** @def GMM_PAGE_IS_PRIVATE
303 *
304 * @returns true if private, false if not.
305 * @param pPage The GMM page.
306 */
307#define GMM_PAGE_IS_PRIVATE(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_PRIVATE )
308
309/** @def GMM_PAGE_IS_SHARED
310 *
311 * @returns true if shared, false if not.
312 * @param pPage The GMM page.
313 */
314#define GMM_PAGE_IS_SHARED(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_SHARED )
315
316/** @def GMM_PAGE_IS_FREE
317 *
318 * @returns true if free, false if not.
319 * @param pPage The GMM page.
320 */
321#define GMM_PAGE_IS_FREE(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_FREE )
322
323/** @def GMM_PAGE_PFN_LAST
324 * The last valid guest pfn range.
325 * @remark Some of the values outside the range has special meaning,
326 * see GMM_PAGE_PFN_UNSHAREABLE.
327 */
328#define GMM_PAGE_PFN_LAST UINT32_C(0xfffffff0)
329AssertCompile(GMM_PAGE_PFN_LAST == (GMM_GCPHYS_LAST >> GUEST_PAGE_SHIFT));
330
331/** @def GMM_PAGE_PFN_UNSHAREABLE
332 * Indicates that this page isn't used for normal guest memory and thus isn't shareable.
333 */
334#define GMM_PAGE_PFN_UNSHAREABLE UINT32_C(0xfffffff1)
335AssertCompile(GMM_PAGE_PFN_UNSHAREABLE == (GMM_GCPHYS_UNSHAREABLE >> GUEST_PAGE_SHIFT));
336
337
338/**
339 * A GMM allocation chunk ring-3 mapping record.
340 *
341 * This should really be associated with a session and not a VM, but
342 * it's simpler to associated with a VM and cleanup with the VM object
343 * is destroyed.
344 */
345typedef struct GMMCHUNKMAP
346{
347 /** The mapping object. */
348 RTR0MEMOBJ hMapObj;
349 /** The VM owning the mapping. */
350 PGVM pGVM;
351} GMMCHUNKMAP;
352/** Pointer to a GMM allocation chunk mapping. */
353typedef struct GMMCHUNKMAP *PGMMCHUNKMAP;
354
355
356/**
357 * A GMM allocation chunk.
358 */
359typedef struct GMMCHUNK
360{
361 /** The AVL node core.
362 * The Key is the chunk ID. (Giant mtx.) */
363 AVLU32NODECORE Core;
364 /** The memory object.
365 * Either from RTR0MemObjAllocPhysNC or RTR0MemObjLockUser depending on
366 * what the host can dish up with. (Chunk mtx protects mapping accesses
367 * and related frees.) */
368 RTR0MEMOBJ hMemObj;
369#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
370 /** Pointer to the kernel mapping. */
371 uint8_t *pbMapping;
372#endif
373 /** Pointer to the next chunk in the free list. (Giant mtx.) */
374 PGMMCHUNK pFreeNext;
375 /** Pointer to the previous chunk in the free list. (Giant mtx.) */
376 PGMMCHUNK pFreePrev;
377 /** Pointer to the free set this chunk belongs to. NULL for
378 * chunks with no free pages. (Giant mtx.) */
379 PGMMCHUNKFREESET pSet;
380 /** List node in the chunk list (GMM::ChunkList). (Giant mtx.) */
381 RTLISTNODE ListNode;
382 /** Pointer to an array of mappings. (Chunk mtx.) */
383 PGMMCHUNKMAP paMappingsX;
384 /** The number of mappings. (Chunk mtx.) */
385 uint16_t cMappingsX;
386 /** The mapping lock this chunk is using using. UINT8_MAX if nobody is mapping
387 * or freeing anything. (Giant mtx.) */
388 uint8_t volatile iChunkMtx;
389 /** GMM_CHUNK_FLAGS_XXX. (Giant mtx.) */
390 uint8_t fFlags;
391 /** The head of the list of free pages. UINT16_MAX is the NIL value.
392 * (Giant mtx.) */
393 uint16_t iFreeHead;
394 /** The number of free pages. (Giant mtx.) */
395 uint16_t cFree;
396 /** The GVM handle of the VM that first allocated pages from this chunk, this
397 * is used as a preference when there are several chunks to choose from.
398 * When in bound memory mode this isn't a preference any longer. (Giant
399 * mtx.) */
400 uint16_t hGVM;
401 /** The ID of the NUMA node the memory mostly resides on. (Reserved for
402 * future use.) (Giant mtx.) */
403 uint16_t idNumaNode;
404 /** The number of private pages. (Giant mtx.) */
405 uint16_t cPrivate;
406 /** The number of shared pages. (Giant mtx.) */
407 uint16_t cShared;
408 /** The UID this chunk is associated with. */
409 RTUID uidOwner;
410 uint32_t u32Padding;
411 /** The pages. (Giant mtx.) */
412 GMMPAGE aPages[GMM_CHUNK_NUM_PAGES];
413} GMMCHUNK;
414
415/** Indicates that the NUMA properies of the memory is unknown. */
416#define GMM_CHUNK_NUMA_ID_UNKNOWN UINT16_C(0xfffe)
417
418/** @name GMM_CHUNK_FLAGS_XXX - chunk flags.
419 * @{ */
420/** Indicates that the chunk is a large page (2MB). */
421#define GMM_CHUNK_FLAGS_LARGE_PAGE UINT16_C(0x0001)
422/** @} */
423
424
425/**
426 * An allocation chunk TLB entry.
427 */
428typedef struct GMMCHUNKTLBE
429{
430 /** The chunk id. */
431 uint32_t idChunk;
432 /** Pointer to the chunk. */
433 PGMMCHUNK pChunk;
434} GMMCHUNKTLBE;
435/** Pointer to an allocation chunk TLB entry. */
436typedef GMMCHUNKTLBE *PGMMCHUNKTLBE;
437
438
439/** The number of entries in the allocation chunk TLB. */
440#define GMM_CHUNKTLB_ENTRIES 32
441/** Gets the TLB entry index for the given Chunk ID. */
442#define GMM_CHUNKTLB_IDX(idChunk) ( (idChunk) & (GMM_CHUNKTLB_ENTRIES - 1) )
443
444/**
445 * An allocation chunk TLB.
446 */
447typedef struct GMMCHUNKTLB
448{
449 /** The TLB entries. */
450 GMMCHUNKTLBE aEntries[GMM_CHUNKTLB_ENTRIES];
451} GMMCHUNKTLB;
452/** Pointer to an allocation chunk TLB. */
453typedef GMMCHUNKTLB *PGMMCHUNKTLB;
454
455
456/**
457 * The GMM instance data.
458 */
459typedef struct GMM
460{
461 /** Magic / eye catcher. GMM_MAGIC */
462 uint32_t u32Magic;
463 /** The number of threads waiting on the mutex. */
464 uint32_t cMtxContenders;
465#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
466 /** The critical section protecting the GMM.
467 * More fine grained locking can be implemented later if necessary. */
468 RTCRITSECT GiantCritSect;
469#else
470 /** The fast mutex protecting the GMM.
471 * More fine grained locking can be implemented later if necessary. */
472 RTSEMFASTMUTEX hMtx;
473#endif
474#ifdef VBOX_STRICT
475 /** The current mutex owner. */
476 RTNATIVETHREAD hMtxOwner;
477#endif
478 /** Spinlock protecting the AVL tree.
479 * @todo Make this a read-write spinlock as we should allow concurrent
480 * lookups. */
481 RTSPINLOCK hSpinLockTree;
482 /** The chunk tree.
483 * Protected by hSpinLockTree. */
484 PAVLU32NODECORE pChunks;
485 /** Chunk freeing generation - incremented whenever a chunk is freed. Used
486 * for validating the per-VM chunk TLB entries. Valid range is 1 to 2^62
487 * (exclusive), though higher numbers may temporarily occure while
488 * invalidating the individual TLBs during wrap-around processing. */
489 uint64_t volatile idFreeGeneration;
490 /** The chunk TLB.
491 * Protected by hSpinLockTree. */
492 GMMCHUNKTLB ChunkTLB;
493 /** The private free set. */
494 GMMCHUNKFREESET PrivateX;
495 /** The shared free set. */
496 GMMCHUNKFREESET Shared;
497
498 /** Shared module tree (global).
499 * @todo separate trees for distinctly different guest OSes. */
500 PAVLLU32NODECORE pGlobalSharedModuleTree;
501 /** Sharable modules (count of nodes in pGlobalSharedModuleTree). */
502 uint32_t cShareableModules;
503
504 /** The chunk list. For simplifying the cleanup process and avoid tree
505 * traversal. */
506 RTLISTANCHOR ChunkList;
507
508 /** The maximum number of pages we're allowed to allocate.
509 * @gcfgm{GMM/MaxPages,64-bit, Direct.}
510 * @gcfgm{GMM/PctPages,32-bit, Relative to the number of host pages.} */
511 uint64_t cMaxPages;
512 /** The number of pages that has been reserved.
513 * The deal is that cReservedPages - cOverCommittedPages <= cMaxPages. */
514 uint64_t cReservedPages;
515 /** The number of pages that we have over-committed in reservations. */
516 uint64_t cOverCommittedPages;
517 /** The number of actually allocated (committed if you like) pages. */
518 uint64_t cAllocatedPages;
519 /** The number of pages that are shared. A subset of cAllocatedPages. */
520 uint64_t cSharedPages;
521 /** The number of pages that are actually shared between VMs. */
522 uint64_t cDuplicatePages;
523 /** The number of pages that are shared that has been left behind by
524 * VMs not doing proper cleanups. */
525 uint64_t cLeftBehindSharedPages;
526 /** The number of allocation chunks.
527 * (The number of pages we've allocated from the host can be derived from this.) */
528 uint32_t cChunks;
529 /** The number of current ballooned pages. */
530 uint64_t cBalloonedPages;
531
532#ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
533 /** Whether #RTR0MemObjAllocPhysNC works. */
534 bool fHasWorkingAllocPhysNC;
535#else
536 bool fPadding;
537#endif
538 /** The bound memory mode indicator.
539 * When set, the memory will be bound to a specific VM and never
540 * shared. This is always set if fLegacyAllocationMode is set.
541 * (Also determined at initialization time.) */
542 bool fBoundMemoryMode;
543 /** The number of registered VMs. */
544 uint16_t cRegisteredVMs;
545
546 /** The index of the next mutex to use. */
547 uint32_t iNextChunkMtx;
548 /** Chunk locks for reducing lock contention without having to allocate
549 * one lock per chunk. */
550 struct
551 {
552 /** The mutex */
553 RTSEMFASTMUTEX hMtx;
554 /** The number of threads currently using this mutex. */
555 uint32_t volatile cUsers;
556 } aChunkMtx[64];
557
558 /** The number of freed chunks ever. This is used as list generation to
559 * avoid restarting the cleanup scanning when the list wasn't modified. */
560 uint32_t volatile cFreedChunks;
561 /** The previous allocated Chunk ID.
562 * Used as a hint to avoid scanning the whole bitmap. */
563 uint32_t idChunkPrev;
564 /** Spinlock protecting idChunkPrev & bmChunkId. */
565 RTSPINLOCK hSpinLockChunkId;
566 /** Chunk ID allocation bitmap.
567 * Bits of allocated IDs are set, free ones are clear.
568 * The NIL id (0) is marked allocated. */
569 uint32_t bmChunkId[(GMM_CHUNKID_LAST + 1 + 31) / 32];
570} GMM;
571/** Pointer to the GMM instance. */
572typedef GMM *PGMM;
573
574/** The value of GMM::u32Magic (Katsuhiro Otomo). */
575#define GMM_MAGIC UINT32_C(0x19540414)
576
577
578/**
579 * GMM chunk mutex state.
580 *
581 * This is returned by gmmR0ChunkMutexAcquire and is used by the other
582 * gmmR0ChunkMutex* methods.
583 */
584typedef struct GMMR0CHUNKMTXSTATE
585{
586 PGMM pGMM;
587 /** The index of the chunk mutex. */
588 uint8_t iChunkMtx;
589 /** The relevant flags (GMMR0CHUNK_MTX_XXX). */
590 uint8_t fFlags;
591} GMMR0CHUNKMTXSTATE;
592/** Pointer to a chunk mutex state. */
593typedef GMMR0CHUNKMTXSTATE *PGMMR0CHUNKMTXSTATE;
594
595/** @name GMMR0CHUNK_MTX_XXX
596 * @{ */
597#define GMMR0CHUNK_MTX_INVALID UINT32_C(0)
598#define GMMR0CHUNK_MTX_KEEP_GIANT UINT32_C(1)
599#define GMMR0CHUNK_MTX_RETAKE_GIANT UINT32_C(2)
600#define GMMR0CHUNK_MTX_DROP_GIANT UINT32_C(3)
601#define GMMR0CHUNK_MTX_END UINT32_C(4)
602/** @} */
603
604
605/** The maximum number of shared modules per-vm. */
606#define GMM_MAX_SHARED_PER_VM_MODULES 2048
607/** The maximum number of shared modules GMM is allowed to track. */
608#define GMM_MAX_SHARED_GLOBAL_MODULES 16834
609
610
611/**
612 * Argument packet for gmmR0SharedModuleCleanup.
613 */
614typedef struct GMMR0SHMODPERVMDTORARGS
615{
616 PGVM pGVM;
617 PGMM pGMM;
618} GMMR0SHMODPERVMDTORARGS;
619
620/**
621 * Argument packet for gmmR0CheckSharedModule.
622 */
623typedef struct GMMCHECKSHAREDMODULEINFO
624{
625 PGVM pGVM;
626 VMCPUID idCpu;
627} GMMCHECKSHAREDMODULEINFO;
628
629
630/*********************************************************************************************************************************
631* Global Variables *
632*********************************************************************************************************************************/
633/** Pointer to the GMM instance data. */
634static PGMM g_pGMM = NULL;
635
636/** Macro for obtaining and validating the g_pGMM pointer.
637 *
638 * On failure it will return from the invoking function with the specified
639 * return value.
640 *
641 * @param pGMM The name of the pGMM variable.
642 * @param rc The return value on failure. Use VERR_GMM_INSTANCE for VBox
643 * status codes.
644 */
645#define GMM_GET_VALID_INSTANCE(pGMM, rc) \
646 do { \
647 (pGMM) = g_pGMM; \
648 AssertPtrReturn((pGMM), (rc)); \
649 AssertMsgReturn((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic), (rc)); \
650 } while (0)
651
652/** Macro for obtaining and validating the g_pGMM pointer, void function
653 * variant.
654 *
655 * On failure it will return from the invoking function.
656 *
657 * @param pGMM The name of the pGMM variable.
658 */
659#define GMM_GET_VALID_INSTANCE_VOID(pGMM) \
660 do { \
661 (pGMM) = g_pGMM; \
662 AssertPtrReturnVoid((pGMM)); \
663 AssertMsgReturnVoid((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic)); \
664 } while (0)
665
666
667/** @def GMM_CHECK_SANITY_UPON_ENTERING
668 * Checks the sanity of the GMM instance data before making changes.
669 *
670 * This is macro is a stub by default and must be enabled manually in the code.
671 *
672 * @returns true if sane, false if not.
673 * @param pGMM The name of the pGMM variable.
674 */
675#if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0
676# define GMM_CHECK_SANITY_UPON_ENTERING(pGMM) (RT_LIKELY(gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0))
677#else
678# define GMM_CHECK_SANITY_UPON_ENTERING(pGMM) (true)
679#endif
680
681/** @def GMM_CHECK_SANITY_UPON_LEAVING
682 * Checks the sanity of the GMM instance data after making changes.
683 *
684 * This is macro is a stub by default and must be enabled manually in the code.
685 *
686 * @returns true if sane, false if not.
687 * @param pGMM The name of the pGMM variable.
688 */
689#if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0
690# define GMM_CHECK_SANITY_UPON_LEAVING(pGMM) (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
691#else
692# define GMM_CHECK_SANITY_UPON_LEAVING(pGMM) (true)
693#endif
694
695/** @def GMM_CHECK_SANITY_IN_LOOPS
696 * Checks the sanity of the GMM instance in the allocation loops.
697 *
698 * This is macro is a stub by default and must be enabled manually in the code.
699 *
700 * @returns true if sane, false if not.
701 * @param pGMM The name of the pGMM variable.
702 */
703#if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0
704# define GMM_CHECK_SANITY_IN_LOOPS(pGMM) (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)
705#else
706# define GMM_CHECK_SANITY_IN_LOOPS(pGMM) (true)
707#endif
708
709
710/*********************************************************************************************************************************
711* Internal Functions *
712*********************************************************************************************************************************/
713static DECLCALLBACK(int) gmmR0TermDestroyChunk(PAVLU32NODECORE pNode, void *pvGMM);
714static bool gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
715DECLINLINE(void) gmmR0UnlinkChunk(PGMMCHUNK pChunk);
716DECLINLINE(void) gmmR0LinkChunk(PGMMCHUNK pChunk, PGMMCHUNKFREESET pSet);
717DECLINLINE(void) gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
718#ifdef GMMR0_WITH_SANITY_CHECK
719static uint32_t gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo);
720#endif
721static bool gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem);
722DECLINLINE(void) gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage);
723DECLINLINE(void) gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage);
724static int gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk);
725#ifdef VBOX_WITH_PAGE_SHARING
726static void gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM);
727# ifdef VBOX_STRICT
728static uint32_t gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage);
729# endif
730#endif
731
732
733
734/**
735 * Initializes the GMM component.
736 *
737 * This is called when the VMMR0.r0 module is loaded and protected by the
738 * loader semaphore.
739 *
740 * @returns VBox status code.
741 */
742GMMR0DECL(int) GMMR0Init(void)
743{
744 LogFlow(("GMMInit:\n"));
745
746 /* Currently assuming same host and guest page size here. Can change it to
747 dish out guest pages with different size from the host page later if
748 needed, though a restriction would be the host page size must be larger
749 than the guest page size. */
750 AssertCompile(GUEST_PAGE_SIZE == HOST_PAGE_SIZE);
751 AssertCompile(GUEST_PAGE_SIZE <= HOST_PAGE_SIZE);
752
753 /*
754 * Allocate the instance data and the locks.
755 */
756 PGMM pGMM = (PGMM)RTMemAllocZ(sizeof(*pGMM));
757 if (!pGMM)
758 return VERR_NO_MEMORY;
759
760 pGMM->u32Magic = GMM_MAGIC;
761 for (unsigned i = 0; i < RT_ELEMENTS(pGMM->ChunkTLB.aEntries); i++)
762 pGMM->ChunkTLB.aEntries[i].idChunk = NIL_GMM_CHUNKID;
763 RTListInit(&pGMM->ChunkList);
764 ASMBitSet(&pGMM->bmChunkId[0], NIL_GMM_CHUNKID);
765
766#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
767 int rc = RTCritSectInit(&pGMM->GiantCritSect);
768#else
769 int rc = RTSemFastMutexCreate(&pGMM->hMtx);
770#endif
771 if (RT_SUCCESS(rc))
772 {
773 unsigned iMtx;
774 for (iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++)
775 {
776 rc = RTSemFastMutexCreate(&pGMM->aChunkMtx[iMtx].hMtx);
777 if (RT_FAILURE(rc))
778 break;
779 }
780 pGMM->hSpinLockTree = NIL_RTSPINLOCK;
781 if (RT_SUCCESS(rc))
782 rc = RTSpinlockCreate(&pGMM->hSpinLockTree, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "gmm-chunk-tree");
783 pGMM->hSpinLockChunkId = NIL_RTSPINLOCK;
784 if (RT_SUCCESS(rc))
785 rc = RTSpinlockCreate(&pGMM->hSpinLockChunkId, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "gmm-chunk-id");
786 if (RT_SUCCESS(rc))
787 {
788 /*
789 * Figure out how we're going to allocate stuff (only applicable to
790 * host with linear physical memory mappings).
791 */
792 pGMM->fBoundMemoryMode = false;
793#ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
794 pGMM->fHasWorkingAllocPhysNC = false;
795
796 RTR0MEMOBJ hMemObj;
797 rc = RTR0MemObjAllocPhysNC(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS);
798 if (RT_SUCCESS(rc))
799 {
800 rc = RTR0MemObjFree(hMemObj, true);
801 AssertRC(rc);
802 pGMM->fHasWorkingAllocPhysNC = true;
803 }
804 else if (rc != VERR_NOT_SUPPORTED)
805 SUPR0Printf("GMMR0Init: Warning! RTR0MemObjAllocPhysNC(, %u, NIL_RTHCPHYS) -> %d!\n", GMM_CHUNK_SIZE, rc);
806# endif
807
808 /*
809 * Query system page count and guess a reasonable cMaxPages value.
810 */
811 pGMM->cMaxPages = UINT32_MAX; /** @todo IPRT function for query ram size and such. */
812
813 /*
814 * The idFreeGeneration value should be set so we actually trigger the
815 * wrap-around invalidation handling during a typical test run.
816 */
817 pGMM->idFreeGeneration = UINT64_MAX / 4 - 128;
818
819 g_pGMM = pGMM;
820#ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
821 LogFlow(("GMMInit: pGMM=%p fBoundMemoryMode=%RTbool fHasWorkingAllocPhysNC=%RTbool\n", pGMM, pGMM->fBoundMemoryMode, pGMM->fHasWorkingAllocPhysNC));
822#else
823 LogFlow(("GMMInit: pGMM=%p fBoundMemoryMode=%RTbool\n", pGMM, pGMM->fBoundMemoryMode));
824#endif
825 return VINF_SUCCESS;
826 }
827
828 /*
829 * Bail out.
830 */
831 RTSpinlockDestroy(pGMM->hSpinLockChunkId);
832 RTSpinlockDestroy(pGMM->hSpinLockTree);
833 while (iMtx-- > 0)
834 RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx);
835#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
836 RTCritSectDelete(&pGMM->GiantCritSect);
837#else
838 RTSemFastMutexDestroy(pGMM->hMtx);
839#endif
840 }
841
842 pGMM->u32Magic = 0;
843 RTMemFree(pGMM);
844 SUPR0Printf("GMMR0Init: failed! rc=%d\n", rc);
845 return rc;
846}
847
848
849/**
850 * Terminates the GMM component.
851 */
852GMMR0DECL(void) GMMR0Term(void)
853{
854 LogFlow(("GMMTerm:\n"));
855
856 /*
857 * Take care / be paranoid...
858 */
859 PGMM pGMM = g_pGMM;
860 if (!RT_VALID_PTR(pGMM))
861 return;
862 if (pGMM->u32Magic != GMM_MAGIC)
863 {
864 SUPR0Printf("GMMR0Term: u32Magic=%#x\n", pGMM->u32Magic);
865 return;
866 }
867
868 /*
869 * Undo what init did and free all the resources we've acquired.
870 */
871 /* Destroy the fundamentals. */
872 g_pGMM = NULL;
873 pGMM->u32Magic = ~GMM_MAGIC;
874#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
875 RTCritSectDelete(&pGMM->GiantCritSect);
876#else
877 RTSemFastMutexDestroy(pGMM->hMtx);
878 pGMM->hMtx = NIL_RTSEMFASTMUTEX;
879#endif
880 RTSpinlockDestroy(pGMM->hSpinLockTree);
881 pGMM->hSpinLockTree = NIL_RTSPINLOCK;
882 RTSpinlockDestroy(pGMM->hSpinLockChunkId);
883 pGMM->hSpinLockChunkId = NIL_RTSPINLOCK;
884
885 /* Free any chunks still hanging around. */
886 RTAvlU32Destroy(&pGMM->pChunks, gmmR0TermDestroyChunk, pGMM);
887
888 /* Destroy the chunk locks. */
889 for (unsigned iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++)
890 {
891 Assert(pGMM->aChunkMtx[iMtx].cUsers == 0);
892 RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx);
893 pGMM->aChunkMtx[iMtx].hMtx = NIL_RTSEMFASTMUTEX;
894 }
895
896 /* Finally the instance data itself. */
897 RTMemFree(pGMM);
898 LogFlow(("GMMTerm: done\n"));
899}
900
901
902/**
903 * RTAvlU32Destroy callback.
904 *
905 * @returns 0
906 * @param pNode The node to destroy.
907 * @param pvGMM The GMM handle.
908 */
909static DECLCALLBACK(int) gmmR0TermDestroyChunk(PAVLU32NODECORE pNode, void *pvGMM)
910{
911 PGMMCHUNK pChunk = (PGMMCHUNK)pNode;
912
913 if (pChunk->cFree != GMM_CHUNK_NUM_PAGES)
914 SUPR0Printf("GMMR0Term: %RKv/%#x: cFree=%d cPrivate=%d cShared=%d cMappings=%d\n", pChunk,
915 pChunk->Core.Key, pChunk->cFree, pChunk->cPrivate, pChunk->cShared, pChunk->cMappingsX);
916
917 int rc = RTR0MemObjFree(pChunk->hMemObj, true /* fFreeMappings */);
918 if (RT_FAILURE(rc))
919 {
920 SUPR0Printf("GMMR0Term: %RKv/%#x: RTRMemObjFree(%RKv,true) -> %d (cMappings=%d)\n", pChunk,
921 pChunk->Core.Key, pChunk->hMemObj, rc, pChunk->cMappingsX);
922 AssertRC(rc);
923 }
924 pChunk->hMemObj = NIL_RTR0MEMOBJ;
925
926 RTMemFree(pChunk->paMappingsX);
927 pChunk->paMappingsX = NULL;
928
929 RTMemFree(pChunk);
930 NOREF(pvGMM);
931 return 0;
932}
933
934
935/**
936 * Initializes the per-VM data for the GMM.
937 *
938 * This is called from within the GVMM lock (from GVMMR0CreateVM)
939 * and should only initialize the data members so GMMR0CleanupVM
940 * can deal with them. We reserve no memory or anything here,
941 * that's done later in GMMR0InitVM.
942 *
943 * @param pGVM Pointer to the Global VM structure.
944 */
945GMMR0DECL(int) GMMR0InitPerVMData(PGVM pGVM)
946{
947 AssertCompile(RT_SIZEOFMEMB(GVM,gmm.s) <= RT_SIZEOFMEMB(GVM,gmm.padding));
948
949 pGVM->gmm.s.Stats.enmPolicy = GMMOCPOLICY_INVALID;
950 pGVM->gmm.s.Stats.enmPriority = GMMPRIORITY_INVALID;
951 pGVM->gmm.s.Stats.fMayAllocate = false;
952
953 pGVM->gmm.s.hChunkTlbSpinLock = NIL_RTSPINLOCK;
954 int rc = RTSpinlockCreate(&pGVM->gmm.s.hChunkTlbSpinLock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "per-vm-chunk-tlb");
955 AssertRCReturn(rc, rc);
956
957 return VINF_SUCCESS;
958}
959
960
961/**
962 * Acquires the GMM giant lock.
963 *
964 * @returns Assert status code from RTSemFastMutexRequest.
965 * @param pGMM Pointer to the GMM instance.
966 */
967static int gmmR0MutexAcquire(PGMM pGMM)
968{
969 ASMAtomicIncU32(&pGMM->cMtxContenders);
970#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
971 int rc = RTCritSectEnter(&pGMM->GiantCritSect);
972#else
973 int rc = RTSemFastMutexRequest(pGMM->hMtx);
974#endif
975 ASMAtomicDecU32(&pGMM->cMtxContenders);
976 AssertRC(rc);
977#ifdef VBOX_STRICT
978 pGMM->hMtxOwner = RTThreadNativeSelf();
979#endif
980 return rc;
981}
982
983
984/**
985 * Releases the GMM giant lock.
986 *
987 * @returns Assert status code from RTSemFastMutexRequest.
988 * @param pGMM Pointer to the GMM instance.
989 */
990static int gmmR0MutexRelease(PGMM pGMM)
991{
992#ifdef VBOX_STRICT
993 pGMM->hMtxOwner = NIL_RTNATIVETHREAD;
994#endif
995#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
996 int rc = RTCritSectLeave(&pGMM->GiantCritSect);
997#else
998 int rc = RTSemFastMutexRelease(pGMM->hMtx);
999 AssertRC(rc);
1000#endif
1001 return rc;
1002}
1003
1004
1005/**
1006 * Yields the GMM giant lock if there is contention and a certain minimum time
1007 * has elapsed since we took it.
1008 *
1009 * @returns @c true if the mutex was yielded, @c false if not.
1010 * @param pGMM Pointer to the GMM instance.
1011 * @param puLockNanoTS Where the lock acquisition time stamp is kept
1012 * (in/out).
1013 */
1014static bool gmmR0MutexYield(PGMM pGMM, uint64_t *puLockNanoTS)
1015{
1016 /*
1017 * If nobody is contending the mutex, don't bother checking the time.
1018 */
1019 if (ASMAtomicReadU32(&pGMM->cMtxContenders) == 0)
1020 return false;
1021
1022 /*
1023 * Don't yield if we haven't executed for at least 2 milliseconds.
1024 */
1025 uint64_t uNanoNow = RTTimeSystemNanoTS();
1026 if (uNanoNow - *puLockNanoTS < UINT32_C(2000000))
1027 return false;
1028
1029 /*
1030 * Yield the mutex.
1031 */
1032#ifdef VBOX_STRICT
1033 pGMM->hMtxOwner = NIL_RTNATIVETHREAD;
1034#endif
1035 ASMAtomicIncU32(&pGMM->cMtxContenders);
1036#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
1037 int rc1 = RTCritSectLeave(&pGMM->GiantCritSect); AssertRC(rc1);
1038#else
1039 int rc1 = RTSemFastMutexRelease(pGMM->hMtx); AssertRC(rc1);
1040#endif
1041
1042 RTThreadYield();
1043
1044#ifdef VBOX_USE_CRIT_SECT_FOR_GIANT
1045 int rc2 = RTCritSectEnter(&pGMM->GiantCritSect); AssertRC(rc2);
1046#else
1047 int rc2 = RTSemFastMutexRequest(pGMM->hMtx); AssertRC(rc2);
1048#endif
1049 *puLockNanoTS = RTTimeSystemNanoTS();
1050 ASMAtomicDecU32(&pGMM->cMtxContenders);
1051#ifdef VBOX_STRICT
1052 pGMM->hMtxOwner = RTThreadNativeSelf();
1053#endif
1054
1055 return true;
1056}
1057
1058
1059/**
1060 * Acquires a chunk lock.
1061 *
1062 * The caller must own the giant lock.
1063 *
1064 * @returns Assert status code from RTSemFastMutexRequest.
1065 * @param pMtxState The chunk mutex state info. (Avoids
1066 * passing the same flags and stuff around
1067 * for subsequent release and drop-giant
1068 * calls.)
1069 * @param pGMM Pointer to the GMM instance.
1070 * @param pChunk Pointer to the chunk.
1071 * @param fFlags Flags regarding the giant lock, GMMR0CHUNK_MTX_XXX.
1072 */
1073static int gmmR0ChunkMutexAcquire(PGMMR0CHUNKMTXSTATE pMtxState, PGMM pGMM, PGMMCHUNK pChunk, uint32_t fFlags)
1074{
1075 Assert(fFlags > GMMR0CHUNK_MTX_INVALID && fFlags < GMMR0CHUNK_MTX_END);
1076 Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
1077
1078 pMtxState->pGMM = pGMM;
1079 pMtxState->fFlags = (uint8_t)fFlags;
1080
1081 /*
1082 * Get the lock index and reference the lock.
1083 */
1084 Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
1085 uint32_t iChunkMtx = pChunk->iChunkMtx;
1086 if (iChunkMtx == UINT8_MAX)
1087 {
1088 iChunkMtx = pGMM->iNextChunkMtx++;
1089 iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
1090
1091 /* Try get an unused one... */
1092 if (pGMM->aChunkMtx[iChunkMtx].cUsers)
1093 {
1094 iChunkMtx = pGMM->iNextChunkMtx++;
1095 iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
1096 if (pGMM->aChunkMtx[iChunkMtx].cUsers)
1097 {
1098 iChunkMtx = pGMM->iNextChunkMtx++;
1099 iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
1100 if (pGMM->aChunkMtx[iChunkMtx].cUsers)
1101 {
1102 iChunkMtx = pGMM->iNextChunkMtx++;
1103 iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx);
1104 }
1105 }
1106 }
1107
1108 pChunk->iChunkMtx = iChunkMtx;
1109 }
1110 AssertCompile(RT_ELEMENTS(pGMM->aChunkMtx) < UINT8_MAX);
1111 pMtxState->iChunkMtx = (uint8_t)iChunkMtx;
1112 ASMAtomicIncU32(&pGMM->aChunkMtx[iChunkMtx].cUsers);
1113
1114 /*
1115 * Drop the giant?
1116 */
1117 if (fFlags != GMMR0CHUNK_MTX_KEEP_GIANT)
1118 {
1119 /** @todo GMM life cycle cleanup (we may race someone
1120 * destroying and cleaning up GMM)? */
1121 gmmR0MutexRelease(pGMM);
1122 }
1123
1124 /*
1125 * Take the chunk mutex.
1126 */
1127 int rc = RTSemFastMutexRequest(pGMM->aChunkMtx[iChunkMtx].hMtx);
1128 AssertRC(rc);
1129 return rc;
1130}
1131
1132
1133/**
1134 * Releases the GMM giant lock.
1135 *
1136 * @returns Assert status code from RTSemFastMutexRequest.
1137 * @param pMtxState Pointer to the chunk mutex state.
1138 * @param pChunk Pointer to the chunk if it's still
1139 * alive, NULL if it isn't. This is used to deassociate
1140 * the chunk from the mutex on the way out so a new one
1141 * can be selected next time, thus avoiding contented
1142 * mutexes.
1143 */
1144static int gmmR0ChunkMutexRelease(PGMMR0CHUNKMTXSTATE pMtxState, PGMMCHUNK pChunk)
1145{
1146 PGMM pGMM = pMtxState->pGMM;
1147
1148 /*
1149 * Release the chunk mutex and reacquire the giant if requested.
1150 */
1151 int rc = RTSemFastMutexRelease(pGMM->aChunkMtx[pMtxState->iChunkMtx].hMtx);
1152 AssertRC(rc);
1153 if (pMtxState->fFlags == GMMR0CHUNK_MTX_RETAKE_GIANT)
1154 rc = gmmR0MutexAcquire(pGMM);
1155 else
1156 Assert((pMtxState->fFlags != GMMR0CHUNK_MTX_DROP_GIANT) == (pGMM->hMtxOwner == RTThreadNativeSelf()));
1157
1158 /*
1159 * Drop the chunk mutex user reference and deassociate it from the chunk
1160 * when possible.
1161 */
1162 if ( ASMAtomicDecU32(&pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers) == 0
1163 && pChunk
1164 && RT_SUCCESS(rc) )
1165 {
1166 if (pMtxState->fFlags != GMMR0CHUNK_MTX_DROP_GIANT)
1167 pChunk->iChunkMtx = UINT8_MAX;
1168 else
1169 {
1170 rc = gmmR0MutexAcquire(pGMM);
1171 if (RT_SUCCESS(rc))
1172 {
1173 if (pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers == 0)
1174 pChunk->iChunkMtx = UINT8_MAX;
1175 rc = gmmR0MutexRelease(pGMM);
1176 }
1177 }
1178 }
1179
1180 pMtxState->pGMM = NULL;
1181 return rc;
1182}
1183
1184
1185/**
1186 * Drops the giant GMM lock we kept in gmmR0ChunkMutexAcquire while keeping the
1187 * chunk locked.
1188 *
1189 * This only works if gmmR0ChunkMutexAcquire was called with
1190 * GMMR0CHUNK_MTX_KEEP_GIANT. gmmR0ChunkMutexRelease will retake the giant
1191 * mutex, i.e. behave as if GMMR0CHUNK_MTX_RETAKE_GIANT was used.
1192 *
1193 * @returns VBox status code (assuming success is ok).
1194 * @param pMtxState Pointer to the chunk mutex state.
1195 */
1196static int gmmR0ChunkMutexDropGiant(PGMMR0CHUNKMTXSTATE pMtxState)
1197{
1198 AssertReturn(pMtxState->fFlags == GMMR0CHUNK_MTX_KEEP_GIANT, VERR_GMM_MTX_FLAGS);
1199 Assert(pMtxState->pGMM->hMtxOwner == RTThreadNativeSelf());
1200 pMtxState->fFlags = GMMR0CHUNK_MTX_RETAKE_GIANT;
1201 /** @todo GMM life cycle cleanup (we may race someone
1202 * destroying and cleaning up GMM)? */
1203 return gmmR0MutexRelease(pMtxState->pGMM);
1204}
1205
1206
1207/**
1208 * For experimenting with NUMA affinity and such.
1209 *
1210 * @returns The current NUMA Node ID.
1211 */
1212static uint16_t gmmR0GetCurrentNumaNodeId(void)
1213{
1214#if 1
1215 return GMM_CHUNK_NUMA_ID_UNKNOWN;
1216#else
1217 return RTMpCpuId() / 16;
1218#endif
1219}
1220
1221
1222
1223/**
1224 * Cleans up when a VM is terminating.
1225 *
1226 * @param pGVM Pointer to the Global VM structure.
1227 */
1228GMMR0DECL(void) GMMR0CleanupVM(PGVM pGVM)
1229{
1230 LogFlow(("GMMR0CleanupVM: pGVM=%p:{.hSelf=%#x}\n", pGVM, pGVM->hSelf));
1231
1232 PGMM pGMM;
1233 GMM_GET_VALID_INSTANCE_VOID(pGMM);
1234
1235#ifdef VBOX_WITH_PAGE_SHARING
1236 /*
1237 * Clean up all registered shared modules first.
1238 */
1239 gmmR0SharedModuleCleanup(pGMM, pGVM);
1240#endif
1241
1242 gmmR0MutexAcquire(pGMM);
1243 uint64_t uLockNanoTS = RTTimeSystemNanoTS();
1244 GMM_CHECK_SANITY_UPON_ENTERING(pGMM);
1245
1246 /*
1247 * The policy is 'INVALID' until the initial reservation
1248 * request has been serviced.
1249 */
1250 if ( pGVM->gmm.s.Stats.enmPolicy > GMMOCPOLICY_INVALID
1251 && pGVM->gmm.s.Stats.enmPolicy < GMMOCPOLICY_END)
1252 {
1253 /*
1254 * If it's the last VM around, we can skip walking all the chunk looking
1255 * for the pages owned by this VM and instead flush the whole shebang.
1256 *
1257 * This takes care of the eventuality that a VM has left shared page
1258 * references behind (shouldn't happen of course, but you never know).
1259 */
1260 Assert(pGMM->cRegisteredVMs);
1261 pGMM->cRegisteredVMs--;
1262
1263 /*
1264 * Walk the entire pool looking for pages that belong to this VM
1265 * and leftover mappings. (This'll only catch private pages,
1266 * shared pages will be 'left behind'.)
1267 */
1268 /** @todo r=bird: This scanning+freeing could be optimized in bound mode! */
1269 uint64_t cPrivatePages = pGVM->gmm.s.Stats.cPrivatePages; /* save */
1270
1271 unsigned iCountDown = 64;
1272 bool fRedoFromStart;
1273 PGMMCHUNK pChunk;
1274 do
1275 {
1276 fRedoFromStart = false;
1277 RTListForEachReverse(&pGMM->ChunkList, pChunk, GMMCHUNK, ListNode)
1278 {
1279 uint32_t const cFreeChunksOld = pGMM->cFreedChunks;
1280 if ( ( !pGMM->fBoundMemoryMode
1281 || pChunk->hGVM == pGVM->hSelf)
1282 && gmmR0CleanupVMScanChunk(pGMM, pGVM, pChunk))
1283 {
1284 /* We left the giant mutex, so reset the yield counters. */
1285 uLockNanoTS = RTTimeSystemNanoTS();
1286 iCountDown = 64;
1287 }
1288 else
1289 {
1290 /* Didn't leave it, so do normal yielding. */
1291 if (!iCountDown)
1292 gmmR0MutexYield(pGMM, &uLockNanoTS);
1293 else
1294 iCountDown--;
1295 }
1296 if (pGMM->cFreedChunks != cFreeChunksOld)
1297 {
1298 fRedoFromStart = true;
1299 break;
1300 }
1301 }
1302 } while (fRedoFromStart);
1303
1304 if (pGVM->gmm.s.Stats.cPrivatePages)
1305 SUPR0Printf("GMMR0CleanupVM: hGVM=%#x has %#x private pages that cannot be found!\n", pGVM->hSelf, pGVM->gmm.s.Stats.cPrivatePages);
1306
1307 pGMM->cAllocatedPages -= cPrivatePages;
1308
1309 /*
1310 * Free empty chunks.
1311 */
1312 PGMMCHUNKFREESET pPrivateSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX;
1313 do
1314 {
1315 fRedoFromStart = false;
1316 iCountDown = 10240;
1317 pChunk = pPrivateSet->apLists[GMM_CHUNK_FREE_SET_UNUSED_LIST];
1318 while (pChunk)
1319 {
1320 PGMMCHUNK pNext = pChunk->pFreeNext;
1321 Assert(pChunk->cFree == GMM_CHUNK_NUM_PAGES);
1322 if ( !pGMM->fBoundMemoryMode
1323 || pChunk->hGVM == pGVM->hSelf)
1324 {
1325 uint64_t const idGenerationOld = pPrivateSet->idGeneration;
1326 if (gmmR0FreeChunk(pGMM, pGVM, pChunk, true /*fRelaxedSem*/))
1327 {
1328 /* We've left the giant mutex, restart? (+1 for our unlink) */
1329 fRedoFromStart = pPrivateSet->idGeneration != idGenerationOld + 1;
1330 if (fRedoFromStart)
1331 break;
1332 uLockNanoTS = RTTimeSystemNanoTS();
1333 iCountDown = 10240;
1334 }
1335 }
1336
1337 /* Advance and maybe yield the lock. */
1338 pChunk = pNext;
1339 if (--iCountDown == 0)
1340 {
1341 uint64_t const idGenerationOld = pPrivateSet->idGeneration;
1342 fRedoFromStart = gmmR0MutexYield(pGMM, &uLockNanoTS)
1343 && pPrivateSet->idGeneration != idGenerationOld;
1344 if (fRedoFromStart)
1345 break;
1346 iCountDown = 10240;
1347 }
1348 }
1349 } while (fRedoFromStart);
1350
1351 /*
1352 * Account for shared pages that weren't freed.
1353 */
1354 if (pGVM->gmm.s.Stats.cSharedPages)
1355 {
1356 Assert(pGMM->cSharedPages >= pGVM->gmm.s.Stats.cSharedPages);
1357 SUPR0Printf("GMMR0CleanupVM: hGVM=%#x left %#x shared pages behind!\n", pGVM->hSelf, pGVM->gmm.s.Stats.cSharedPages);
1358 pGMM->cLeftBehindSharedPages += pGVM->gmm.s.Stats.cSharedPages;
1359 }
1360
1361 /*
1362 * Clean up balloon statistics in case the VM process crashed.
1363 */
1364 Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages);
1365 pGMM->cBalloonedPages -= pGVM->gmm.s.Stats.cBalloonedPages;
1366
1367 /*
1368 * Update the over-commitment management statistics.
1369 */
1370 pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages
1371 + pGVM->gmm.s.Stats.Reserved.cFixedPages
1372 + pGVM->gmm.s.Stats.Reserved.cShadowPages;
1373 switch (pGVM->gmm.s.Stats.enmPolicy)
1374 {
1375 case GMMOCPOLICY_NO_OC:
1376 break;
1377 default:
1378 /** @todo Update GMM->cOverCommittedPages */
1379 break;
1380 }
1381 }
1382
1383 /* zap the GVM data. */
1384 pGVM->gmm.s.Stats.enmPolicy = GMMOCPOLICY_INVALID;
1385 pGVM->gmm.s.Stats.enmPriority = GMMPRIORITY_INVALID;
1386 pGVM->gmm.s.Stats.fMayAllocate = false;
1387
1388 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
1389 gmmR0MutexRelease(pGMM);
1390
1391 /*
1392 * Destroy the spinlock.
1393 */
1394 RTSPINLOCK hSpinlock = NIL_RTSPINLOCK;
1395 ASMAtomicXchgHandle(&pGVM->gmm.s.hChunkTlbSpinLock, NIL_RTSPINLOCK, &hSpinlock);
1396 RTSpinlockDestroy(hSpinlock);
1397
1398 LogFlow(("GMMR0CleanupVM: returns\n"));
1399}
1400
1401
1402/**
1403 * Scan one chunk for private pages belonging to the specified VM.
1404 *
1405 * @note This function may drop the giant mutex!
1406 *
1407 * @returns @c true if we've temporarily dropped the giant mutex, @c false if
1408 * we didn't.
1409 * @param pGMM Pointer to the GMM instance.
1410 * @param pGVM The global VM handle.
1411 * @param pChunk The chunk to scan.
1412 */
1413static bool gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
1414{
1415 Assert(!pGMM->fBoundMemoryMode || pChunk->hGVM == pGVM->hSelf);
1416
1417 /*
1418 * Look for pages belonging to the VM.
1419 * (Perform some internal checks while we're scanning.)
1420 */
1421#ifndef VBOX_STRICT
1422 if (pChunk->cFree != GMM_CHUNK_NUM_PAGES)
1423#endif
1424 {
1425 unsigned cPrivate = 0;
1426 unsigned cShared = 0;
1427 unsigned cFree = 0;
1428
1429 gmmR0UnlinkChunk(pChunk); /* avoiding cFreePages updates. */
1430
1431 uint16_t hGVM = pGVM->hSelf;
1432 unsigned iPage = (GMM_CHUNK_SIZE >> GUEST_PAGE_SHIFT);
1433 while (iPage-- > 0)
1434 if (GMM_PAGE_IS_PRIVATE(&pChunk->aPages[iPage]))
1435 {
1436 if (pChunk->aPages[iPage].Private.hGVM == hGVM)
1437 {
1438 /*
1439 * Free the page.
1440 *
1441 * The reason for not using gmmR0FreePrivatePage here is that we
1442 * must *not* cause the chunk to be freed from under us - we're in
1443 * an AVL tree walk here.
1444 */
1445 pChunk->aPages[iPage].u = 0;
1446 pChunk->aPages[iPage].Free.u2State = GMM_PAGE_STATE_FREE;
1447 pChunk->aPages[iPage].Free.fZeroed = false;
1448 pChunk->aPages[iPage].Free.iNext = pChunk->iFreeHead;
1449 pChunk->iFreeHead = iPage;
1450 pChunk->cPrivate--;
1451 pChunk->cFree++;
1452 pGVM->gmm.s.Stats.cPrivatePages--;
1453 cFree++;
1454 }
1455 else
1456 cPrivate++;
1457 }
1458 else if (GMM_PAGE_IS_FREE(&pChunk->aPages[iPage]))
1459 cFree++;
1460 else
1461 cShared++;
1462
1463 gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
1464
1465 /*
1466 * Did it add up?
1467 */
1468 if (RT_UNLIKELY( pChunk->cFree != cFree
1469 || pChunk->cPrivate != cPrivate
1470 || pChunk->cShared != cShared))
1471 {
1472 SUPR0Printf("gmmR0CleanupVMScanChunk: Chunk %RKv/%#x has bogus stats - free=%d/%d private=%d/%d shared=%d/%d\n",
1473 pChunk, pChunk->Core.Key, pChunk->cFree, cFree, pChunk->cPrivate, cPrivate, pChunk->cShared, cShared);
1474 pChunk->cFree = cFree;
1475 pChunk->cPrivate = cPrivate;
1476 pChunk->cShared = cShared;
1477 }
1478 }
1479
1480 /*
1481 * If not in bound memory mode, we should reset the hGVM field
1482 * if it has our handle in it.
1483 */
1484 if (pChunk->hGVM == pGVM->hSelf)
1485 {
1486 if (!g_pGMM->fBoundMemoryMode)
1487 pChunk->hGVM = NIL_GVM_HANDLE;
1488 else if (pChunk->cFree != GMM_CHUNK_NUM_PAGES)
1489 {
1490 SUPR0Printf("gmmR0CleanupVMScanChunk: %RKv/%#x: cFree=%#x - it should be 0 in bound mode!\n",
1491 pChunk, pChunk->Core.Key, pChunk->cFree);
1492 AssertMsgFailed(("%p/%#x: cFree=%#x - it should be 0 in bound mode!\n", pChunk, pChunk->Core.Key, pChunk->cFree));
1493
1494 gmmR0UnlinkChunk(pChunk);
1495 pChunk->cFree = GMM_CHUNK_NUM_PAGES;
1496 gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
1497 }
1498 }
1499
1500 /*
1501 * Look for a mapping belonging to the terminating VM.
1502 */
1503 GMMR0CHUNKMTXSTATE MtxState;
1504 gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
1505 unsigned cMappings = pChunk->cMappingsX;
1506 for (unsigned i = 0; i < cMappings; i++)
1507 if (pChunk->paMappingsX[i].pGVM == pGVM)
1508 {
1509 gmmR0ChunkMutexDropGiant(&MtxState);
1510
1511 RTR0MEMOBJ hMemObj = pChunk->paMappingsX[i].hMapObj;
1512
1513 cMappings--;
1514 if (i < cMappings)
1515 pChunk->paMappingsX[i] = pChunk->paMappingsX[cMappings];
1516 pChunk->paMappingsX[cMappings].pGVM = NULL;
1517 pChunk->paMappingsX[cMappings].hMapObj = NIL_RTR0MEMOBJ;
1518 Assert(pChunk->cMappingsX - 1U == cMappings);
1519 pChunk->cMappingsX = cMappings;
1520
1521 int rc = RTR0MemObjFree(hMemObj, false /* fFreeMappings (NA) */);
1522 if (RT_FAILURE(rc))
1523 {
1524 SUPR0Printf("gmmR0CleanupVMScanChunk: %RKv/%#x: mapping #%x: RTRMemObjFree(%RKv,false) -> %d \n",
1525 pChunk, pChunk->Core.Key, i, hMemObj, rc);
1526 AssertRC(rc);
1527 }
1528
1529 gmmR0ChunkMutexRelease(&MtxState, pChunk);
1530 return true;
1531 }
1532
1533 gmmR0ChunkMutexRelease(&MtxState, pChunk);
1534 return false;
1535}
1536
1537
1538/**
1539 * The initial resource reservations.
1540 *
1541 * This will make memory reservations according to policy and priority. If there aren't
1542 * sufficient resources available to sustain the VM this function will fail and all
1543 * future allocations requests will fail as well.
1544 *
1545 * These are just the initial reservations made very very early during the VM creation
1546 * process and will be adjusted later in the GMMR0UpdateReservation call after the
1547 * ring-3 init has completed.
1548 *
1549 * @returns VBox status code.
1550 * @retval VERR_GMM_MEMORY_RESERVATION_DECLINED
1551 * @retval VERR_GMM_
1552 *
1553 * @param pGVM The global (ring-0) VM structure.
1554 * @param idCpu The VCPU id - must be zero.
1555 * @param cBasePages The number of pages that may be allocated for the base RAM and ROMs.
1556 * This does not include MMIO2 and similar.
1557 * @param cShadowPages The number of pages that may be allocated for shadow paging structures.
1558 * @param cFixedPages The number of pages that may be allocated for fixed objects like the
1559 * hyper heap, MMIO2 and similar.
1560 * @param enmPolicy The OC policy to use on this VM.
1561 * @param enmPriority The priority in an out-of-memory situation.
1562 *
1563 * @thread The creator thread / EMT(0).
1564 */
1565GMMR0DECL(int) GMMR0InitialReservation(PGVM pGVM, VMCPUID idCpu, uint64_t cBasePages, uint32_t cShadowPages,
1566 uint32_t cFixedPages, GMMOCPOLICY enmPolicy, GMMPRIORITY enmPriority)
1567{
1568 LogFlow(("GMMR0InitialReservation: pGVM=%p cBasePages=%#llx cShadowPages=%#x cFixedPages=%#x enmPolicy=%d enmPriority=%d\n",
1569 pGVM, cBasePages, cShadowPages, cFixedPages, enmPolicy, enmPriority));
1570
1571 /*
1572 * Validate, get basics and take the semaphore.
1573 */
1574 AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID);
1575 PGMM pGMM;
1576 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
1577 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
1578 if (RT_FAILURE(rc))
1579 return rc;
1580
1581 AssertReturn(cBasePages, VERR_INVALID_PARAMETER);
1582 AssertReturn(cShadowPages, VERR_INVALID_PARAMETER);
1583 AssertReturn(cFixedPages, VERR_INVALID_PARAMETER);
1584 AssertReturn(enmPolicy > GMMOCPOLICY_INVALID && enmPolicy < GMMOCPOLICY_END, VERR_INVALID_PARAMETER);
1585 AssertReturn(enmPriority > GMMPRIORITY_INVALID && enmPriority < GMMPRIORITY_END, VERR_INVALID_PARAMETER);
1586
1587 gmmR0MutexAcquire(pGMM);
1588 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
1589 {
1590 if ( !pGVM->gmm.s.Stats.Reserved.cBasePages
1591 && !pGVM->gmm.s.Stats.Reserved.cFixedPages
1592 && !pGVM->gmm.s.Stats.Reserved.cShadowPages)
1593 {
1594 /*
1595 * Check if we can accommodate this.
1596 */
1597 /* ... later ... */
1598 if (RT_SUCCESS(rc))
1599 {
1600 /*
1601 * Update the records.
1602 */
1603 pGVM->gmm.s.Stats.Reserved.cBasePages = cBasePages;
1604 pGVM->gmm.s.Stats.Reserved.cFixedPages = cFixedPages;
1605 pGVM->gmm.s.Stats.Reserved.cShadowPages = cShadowPages;
1606 pGVM->gmm.s.Stats.enmPolicy = enmPolicy;
1607 pGVM->gmm.s.Stats.enmPriority = enmPriority;
1608 pGVM->gmm.s.Stats.fMayAllocate = true;
1609
1610 pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages;
1611 pGMM->cRegisteredVMs++;
1612 }
1613 }
1614 else
1615 rc = VERR_WRONG_ORDER;
1616 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
1617 }
1618 else
1619 rc = VERR_GMM_IS_NOT_SANE;
1620 gmmR0MutexRelease(pGMM);
1621 LogFlow(("GMMR0InitialReservation: returns %Rrc\n", rc));
1622 return rc;
1623}
1624
1625
1626/**
1627 * VMMR0 request wrapper for GMMR0InitialReservation.
1628 *
1629 * @returns see GMMR0InitialReservation.
1630 * @param pGVM The global (ring-0) VM structure.
1631 * @param idCpu The VCPU id.
1632 * @param pReq Pointer to the request packet.
1633 */
1634GMMR0DECL(int) GMMR0InitialReservationReq(PGVM pGVM, VMCPUID idCpu, PGMMINITIALRESERVATIONREQ pReq)
1635{
1636 /*
1637 * Validate input and pass it on.
1638 */
1639 AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
1640 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1641 AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
1642
1643 return GMMR0InitialReservation(pGVM, idCpu, pReq->cBasePages, pReq->cShadowPages,
1644 pReq->cFixedPages, pReq->enmPolicy, pReq->enmPriority);
1645}
1646
1647
1648/**
1649 * This updates the memory reservation with the additional MMIO2 and ROM pages.
1650 *
1651 * @returns VBox status code.
1652 * @retval VERR_GMM_MEMORY_RESERVATION_DECLINED
1653 *
1654 * @param pGVM The global (ring-0) VM structure.
1655 * @param idCpu The VCPU id.
1656 * @param cBasePages The number of pages that may be allocated for the base RAM and ROMs.
1657 * This does not include MMIO2 and similar.
1658 * @param cShadowPages The number of pages that may be allocated for shadow paging structures.
1659 * @param cFixedPages The number of pages that may be allocated for fixed objects like the
1660 * hyper heap, MMIO2 and similar.
1661 *
1662 * @thread EMT(idCpu)
1663 */
1664GMMR0DECL(int) GMMR0UpdateReservation(PGVM pGVM, VMCPUID idCpu, uint64_t cBasePages,
1665 uint32_t cShadowPages, uint32_t cFixedPages)
1666{
1667 LogFlow(("GMMR0UpdateReservation: pGVM=%p cBasePages=%#llx cShadowPages=%#x cFixedPages=%#x\n",
1668 pGVM, cBasePages, cShadowPages, cFixedPages));
1669
1670 /*
1671 * Validate, get basics and take the semaphore.
1672 */
1673 PGMM pGMM;
1674 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
1675 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
1676 if (RT_FAILURE(rc))
1677 return rc;
1678
1679 AssertReturn(cBasePages, VERR_INVALID_PARAMETER);
1680 AssertReturn(cShadowPages, VERR_INVALID_PARAMETER);
1681 AssertReturn(cFixedPages, VERR_INVALID_PARAMETER);
1682
1683 gmmR0MutexAcquire(pGMM);
1684 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
1685 {
1686 if ( pGVM->gmm.s.Stats.Reserved.cBasePages
1687 && pGVM->gmm.s.Stats.Reserved.cFixedPages
1688 && pGVM->gmm.s.Stats.Reserved.cShadowPages)
1689 {
1690 /*
1691 * Check if we can accommodate this.
1692 */
1693 /* ... later ... */
1694 if (RT_SUCCESS(rc))
1695 {
1696 /*
1697 * Update the records.
1698 */
1699 pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages
1700 + pGVM->gmm.s.Stats.Reserved.cFixedPages
1701 + pGVM->gmm.s.Stats.Reserved.cShadowPages;
1702 pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages;
1703
1704 pGVM->gmm.s.Stats.Reserved.cBasePages = cBasePages;
1705 pGVM->gmm.s.Stats.Reserved.cFixedPages = cFixedPages;
1706 pGVM->gmm.s.Stats.Reserved.cShadowPages = cShadowPages;
1707 }
1708 }
1709 else
1710 rc = VERR_WRONG_ORDER;
1711 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
1712 }
1713 else
1714 rc = VERR_GMM_IS_NOT_SANE;
1715 gmmR0MutexRelease(pGMM);
1716 LogFlow(("GMMR0UpdateReservation: returns %Rrc\n", rc));
1717 return rc;
1718}
1719
1720
1721/**
1722 * VMMR0 request wrapper for GMMR0UpdateReservation.
1723 *
1724 * @returns see GMMR0UpdateReservation.
1725 * @param pGVM The global (ring-0) VM structure.
1726 * @param idCpu The VCPU id.
1727 * @param pReq Pointer to the request packet.
1728 */
1729GMMR0DECL(int) GMMR0UpdateReservationReq(PGVM pGVM, VMCPUID idCpu, PGMMUPDATERESERVATIONREQ pReq)
1730{
1731 /*
1732 * Validate input and pass it on.
1733 */
1734 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1735 AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
1736
1737 return GMMR0UpdateReservation(pGVM, idCpu, pReq->cBasePages, pReq->cShadowPages, pReq->cFixedPages);
1738}
1739
1740#ifdef GMMR0_WITH_SANITY_CHECK
1741
1742/**
1743 * Performs sanity checks on a free set.
1744 *
1745 * @returns Error count.
1746 *
1747 * @param pGMM Pointer to the GMM instance.
1748 * @param pSet Pointer to the set.
1749 * @param pszSetName The set name.
1750 * @param pszFunction The function from which it was called.
1751 * @param uLine The line number.
1752 */
1753static uint32_t gmmR0SanityCheckSet(PGMM pGMM, PGMMCHUNKFREESET pSet, const char *pszSetName,
1754 const char *pszFunction, unsigned uLineNo)
1755{
1756 uint32_t cErrors = 0;
1757
1758 /*
1759 * Count the free pages in all the chunks and match it against pSet->cFreePages.
1760 */
1761 uint32_t cPages = 0;
1762 for (unsigned i = 0; i < RT_ELEMENTS(pSet->apLists); i++)
1763 {
1764 for (PGMMCHUNK pCur = pSet->apLists[i]; pCur; pCur = pCur->pFreeNext)
1765 {
1766 /** @todo check that the chunk is hash into the right set. */
1767 cPages += pCur->cFree;
1768 }
1769 }
1770 if (RT_UNLIKELY(cPages != pSet->cFreePages))
1771 {
1772 SUPR0Printf("GMM insanity: found %#x pages in the %s set, expected %#x. (%s, line %u)\n",
1773 cPages, pszSetName, pSet->cFreePages, pszFunction, uLineNo);
1774 cErrors++;
1775 }
1776
1777 return cErrors;
1778}
1779
1780
1781/**
1782 * Performs some sanity checks on the GMM while owning lock.
1783 *
1784 * @returns Error count.
1785 *
1786 * @param pGMM Pointer to the GMM instance.
1787 * @param pszFunction The function from which it is called.
1788 * @param uLineNo The line number.
1789 */
1790static uint32_t gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo)
1791{
1792 uint32_t cErrors = 0;
1793
1794 cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->PrivateX, "private", pszFunction, uLineNo);
1795 cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->Shared, "shared", pszFunction, uLineNo);
1796 /** @todo add more sanity checks. */
1797
1798 return cErrors;
1799}
1800
1801#endif /* GMMR0_WITH_SANITY_CHECK */
1802
1803/**
1804 * Looks up a chunk in the tree and fill in the TLB entry for it.
1805 *
1806 * This is not expected to fail and will bitch if it does.
1807 *
1808 * @returns Pointer to the allocation chunk, NULL if not found.
1809 * @param pGMM Pointer to the GMM instance.
1810 * @param idChunk The ID of the chunk to find.
1811 * @param pTlbe Pointer to the TLB entry.
1812 *
1813 * @note Caller owns spinlock.
1814 */
1815static PGMMCHUNK gmmR0GetChunkSlow(PGMM pGMM, uint32_t idChunk, PGMMCHUNKTLBE pTlbe)
1816{
1817 PGMMCHUNK pChunk = (PGMMCHUNK)RTAvlU32Get(&pGMM->pChunks, idChunk);
1818 AssertMsgReturn(pChunk, ("Chunk %#x not found!\n", idChunk), NULL);
1819 pTlbe->idChunk = idChunk;
1820 pTlbe->pChunk = pChunk;
1821 return pChunk;
1822}
1823
1824
1825/**
1826 * Finds a allocation chunk, spin-locked.
1827 *
1828 * This is not expected to fail and will bitch if it does.
1829 *
1830 * @returns Pointer to the allocation chunk, NULL if not found.
1831 * @param pGMM Pointer to the GMM instance.
1832 * @param idChunk The ID of the chunk to find.
1833 */
1834DECLINLINE(PGMMCHUNK) gmmR0GetChunkLocked(PGMM pGMM, uint32_t idChunk)
1835{
1836 /*
1837 * Do a TLB lookup, branch if not in the TLB.
1838 */
1839 PGMMCHUNKTLBE pTlbe = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(idChunk)];
1840 PGMMCHUNK pChunk = pTlbe->pChunk;
1841 if ( pChunk == NULL
1842 || pTlbe->idChunk != idChunk)
1843 pChunk = gmmR0GetChunkSlow(pGMM, idChunk, pTlbe);
1844 return pChunk;
1845}
1846
1847
1848/**
1849 * Finds a allocation chunk.
1850 *
1851 * This is not expected to fail and will bitch if it does.
1852 *
1853 * @returns Pointer to the allocation chunk, NULL if not found.
1854 * @param pGMM Pointer to the GMM instance.
1855 * @param idChunk The ID of the chunk to find.
1856 */
1857DECLINLINE(PGMMCHUNK) gmmR0GetChunk(PGMM pGMM, uint32_t idChunk)
1858{
1859 RTSpinlockAcquire(pGMM->hSpinLockTree);
1860 PGMMCHUNK pChunk = gmmR0GetChunkLocked(pGMM, idChunk);
1861 RTSpinlockRelease(pGMM->hSpinLockTree);
1862 return pChunk;
1863}
1864
1865
1866/**
1867 * Finds a page.
1868 *
1869 * This is not expected to fail and will bitch if it does.
1870 *
1871 * @returns Pointer to the page, NULL if not found.
1872 * @param pGMM Pointer to the GMM instance.
1873 * @param idPage The ID of the page to find.
1874 */
1875DECLINLINE(PGMMPAGE) gmmR0GetPage(PGMM pGMM, uint32_t idPage)
1876{
1877 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
1878 if (RT_LIKELY(pChunk))
1879 return &pChunk->aPages[idPage & GMM_PAGEID_IDX_MASK];
1880 return NULL;
1881}
1882
1883
1884#if 0 /* unused */
1885/**
1886 * Gets the host physical address for a page given by it's ID.
1887 *
1888 * @returns The host physical address or NIL_RTHCPHYS.
1889 * @param pGMM Pointer to the GMM instance.
1890 * @param idPage The ID of the page to find.
1891 */
1892DECLINLINE(RTHCPHYS) gmmR0GetPageHCPhys(PGMM pGMM, uint32_t idPage)
1893{
1894 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
1895 if (RT_LIKELY(pChunk))
1896 return RTR0MemObjGetPagePhysAddr(pChunk->hMemObj, idPage & GMM_PAGEID_IDX_MASK);
1897 return NIL_RTHCPHYS;
1898}
1899#endif /* unused */
1900
1901
1902/**
1903 * Selects the appropriate free list given the number of free pages.
1904 *
1905 * @returns Free list index.
1906 * @param cFree The number of free pages in the chunk.
1907 */
1908DECLINLINE(unsigned) gmmR0SelectFreeSetList(unsigned cFree)
1909{
1910 unsigned iList = cFree >> GMM_CHUNK_FREE_SET_SHIFT;
1911 AssertMsg(iList < RT_SIZEOFMEMB(GMMCHUNKFREESET, apLists) / RT_SIZEOFMEMB(GMMCHUNKFREESET, apLists[0]),
1912 ("%d (%u)\n", iList, cFree));
1913 return iList;
1914}
1915
1916
1917/**
1918 * Unlinks the chunk from the free list it's currently on (if any).
1919 *
1920 * @param pChunk The allocation chunk.
1921 */
1922DECLINLINE(void) gmmR0UnlinkChunk(PGMMCHUNK pChunk)
1923{
1924 PGMMCHUNKFREESET pSet = pChunk->pSet;
1925 if (RT_LIKELY(pSet))
1926 {
1927 pSet->cFreePages -= pChunk->cFree;
1928 pSet->idGeneration++;
1929
1930 PGMMCHUNK pPrev = pChunk->pFreePrev;
1931 PGMMCHUNK pNext = pChunk->pFreeNext;
1932 if (pPrev)
1933 pPrev->pFreeNext = pNext;
1934 else
1935 pSet->apLists[gmmR0SelectFreeSetList(pChunk->cFree)] = pNext;
1936 if (pNext)
1937 pNext->pFreePrev = pPrev;
1938
1939 pChunk->pSet = NULL;
1940 pChunk->pFreeNext = NULL;
1941 pChunk->pFreePrev = NULL;
1942 }
1943 else
1944 {
1945 Assert(!pChunk->pFreeNext);
1946 Assert(!pChunk->pFreePrev);
1947 Assert(!pChunk->cFree);
1948 }
1949}
1950
1951
1952/**
1953 * Links the chunk onto the appropriate free list in the specified free set.
1954 *
1955 * If no free entries, it's not linked into any list.
1956 *
1957 * @param pChunk The allocation chunk.
1958 * @param pSet The free set.
1959 */
1960DECLINLINE(void) gmmR0LinkChunk(PGMMCHUNK pChunk, PGMMCHUNKFREESET pSet)
1961{
1962 Assert(!pChunk->pSet);
1963 Assert(!pChunk->pFreeNext);
1964 Assert(!pChunk->pFreePrev);
1965
1966 if (pChunk->cFree > 0)
1967 {
1968 pChunk->pSet = pSet;
1969 pChunk->pFreePrev = NULL;
1970 unsigned const iList = gmmR0SelectFreeSetList(pChunk->cFree);
1971 pChunk->pFreeNext = pSet->apLists[iList];
1972 if (pChunk->pFreeNext)
1973 pChunk->pFreeNext->pFreePrev = pChunk;
1974 pSet->apLists[iList] = pChunk;
1975
1976 pSet->cFreePages += pChunk->cFree;
1977 pSet->idGeneration++;
1978 }
1979}
1980
1981
1982/**
1983 * Links the chunk onto the appropriate free list in the specified free set.
1984 *
1985 * If no free entries, it's not linked into any list.
1986 *
1987 * @param pGMM Pointer to the GMM instance.
1988 * @param pGVM Pointer to the kernel-only VM instace data.
1989 * @param pChunk The allocation chunk.
1990 */
1991DECLINLINE(void) gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
1992{
1993 PGMMCHUNKFREESET pSet;
1994 if (pGMM->fBoundMemoryMode)
1995 pSet = &pGVM->gmm.s.Private;
1996 else if (pChunk->cShared)
1997 pSet = &pGMM->Shared;
1998 else
1999 pSet = &pGMM->PrivateX;
2000 gmmR0LinkChunk(pChunk, pSet);
2001}
2002
2003
2004/**
2005 * Frees a Chunk ID.
2006 *
2007 * @param pGMM Pointer to the GMM instance.
2008 * @param idChunk The Chunk ID to free.
2009 */
2010static void gmmR0FreeChunkId(PGMM pGMM, uint32_t idChunk)
2011{
2012 AssertReturnVoid(idChunk != NIL_GMM_CHUNKID);
2013 RTSpinlockAcquire(pGMM->hSpinLockChunkId); /* We could probably skip the locking here, I think. */
2014
2015 AssertMsg(ASMBitTest(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk));
2016 ASMAtomicBitClear(&pGMM->bmChunkId[0], idChunk);
2017
2018 RTSpinlockRelease(pGMM->hSpinLockChunkId);
2019}
2020
2021
2022/**
2023 * Allocates a new Chunk ID.
2024 *
2025 * @returns The Chunk ID.
2026 * @param pGMM Pointer to the GMM instance.
2027 */
2028static uint32_t gmmR0AllocateChunkId(PGMM pGMM)
2029{
2030 AssertCompile(!((GMM_CHUNKID_LAST + 1) & 31)); /* must be a multiple of 32 */
2031 AssertCompile(NIL_GMM_CHUNKID == 0);
2032
2033 RTSpinlockAcquire(pGMM->hSpinLockChunkId);
2034
2035 /*
2036 * Try the next sequential one.
2037 */
2038 int32_t idChunk = ++pGMM->idChunkPrev;
2039 if ( (uint32_t)idChunk <= GMM_CHUNKID_LAST
2040 && idChunk > NIL_GMM_CHUNKID)
2041 {
2042 if (!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk))
2043 {
2044 RTSpinlockRelease(pGMM->hSpinLockChunkId);
2045 return idChunk;
2046 }
2047
2048 /*
2049 * Scan sequentially from the last one.
2050 */
2051 if ((uint32_t)idChunk < GMM_CHUNKID_LAST)
2052 {
2053 idChunk = ASMBitNextClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1, idChunk);
2054 if ( idChunk > NIL_GMM_CHUNKID
2055 && (uint32_t)idChunk <= GMM_CHUNKID_LAST)
2056 {
2057 AssertMsgReturnStmt(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk),
2058 RTSpinlockRelease(pGMM->hSpinLockChunkId), NIL_GMM_CHUNKID);
2059
2060 pGMM->idChunkPrev = idChunk;
2061 RTSpinlockRelease(pGMM->hSpinLockChunkId);
2062 return idChunk;
2063 }
2064 }
2065 }
2066
2067 /*
2068 * Ok, scan from the start.
2069 * We're not racing anyone, so there is no need to expect failures or have restart loops.
2070 */
2071 idChunk = ASMBitFirstClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1);
2072 AssertMsgReturnStmt(idChunk > NIL_GMM_CHUNKID && (uint32_t)idChunk <= GMM_CHUNKID_LAST, ("%#x\n", idChunk),
2073 RTSpinlockRelease(pGMM->hSpinLockChunkId), NIL_GVM_HANDLE);
2074 AssertMsgReturnStmt(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk),
2075 RTSpinlockRelease(pGMM->hSpinLockChunkId), NIL_GMM_CHUNKID);
2076
2077 pGMM->idChunkPrev = idChunk;
2078 RTSpinlockRelease(pGMM->hSpinLockChunkId);
2079 return idChunk;
2080}
2081
2082
2083/**
2084 * Allocates one private page.
2085 *
2086 * Worker for gmmR0AllocatePages.
2087 *
2088 * @param pChunk The chunk to allocate it from.
2089 * @param hGVM The GVM handle of the VM requesting memory.
2090 * @param pPageDesc The page descriptor.
2091 */
2092static void gmmR0AllocatePage(PGMMCHUNK pChunk, uint32_t hGVM, PGMMPAGEDESC pPageDesc)
2093{
2094 /* update the chunk stats. */
2095 if (pChunk->hGVM == NIL_GVM_HANDLE)
2096 pChunk->hGVM = hGVM;
2097 Assert(pChunk->cFree);
2098 pChunk->cFree--;
2099 pChunk->cPrivate++;
2100
2101 /* unlink the first free page. */
2102 const uint32_t iPage = pChunk->iFreeHead;
2103 AssertReleaseMsg(iPage < RT_ELEMENTS(pChunk->aPages), ("%d\n", iPage));
2104 PGMMPAGE pPage = &pChunk->aPages[iPage];
2105 Assert(GMM_PAGE_IS_FREE(pPage));
2106 pChunk->iFreeHead = pPage->Free.iNext;
2107 Log3(("A pPage=%p iPage=%#x/%#x u2State=%d iFreeHead=%#x iNext=%#x\n",
2108 pPage, iPage, (pChunk->Core.Key << GMM_CHUNKID_SHIFT) | iPage,
2109 pPage->Common.u2State, pChunk->iFreeHead, pPage->Free.iNext));
2110
2111 bool const fZeroed = pPage->Free.fZeroed;
2112
2113 /* make the page private. */
2114 pPage->u = 0;
2115 AssertCompile(GMM_PAGE_STATE_PRIVATE == 0);
2116 pPage->Private.hGVM = hGVM;
2117 AssertCompile(NIL_RTHCPHYS >= GMM_GCPHYS_LAST);
2118 AssertCompile(GMM_GCPHYS_UNSHAREABLE >= GMM_GCPHYS_LAST);
2119 if (pPageDesc->HCPhysGCPhys <= GMM_GCPHYS_LAST)
2120 pPage->Private.pfn = pPageDesc->HCPhysGCPhys >> GUEST_PAGE_SHIFT;
2121 else
2122 pPage->Private.pfn = GMM_PAGE_PFN_UNSHAREABLE; /* unshareable / unassigned - same thing. */
2123
2124 /* update the page descriptor. */
2125 pPageDesc->idSharedPage = NIL_GMM_PAGEID;
2126 pPageDesc->idPage = (pChunk->Core.Key << GMM_CHUNKID_SHIFT) | iPage;
2127 RTHCPHYS const HCPhys = RTR0MemObjGetPagePhysAddr(pChunk->hMemObj, iPage);
2128 Assert(HCPhys != NIL_RTHCPHYS); Assert(HCPhys < NIL_GMMPAGEDESC_PHYS);
2129 pPageDesc->HCPhysGCPhys = HCPhys;
2130 pPageDesc->fZeroed = fZeroed;
2131}
2132
2133
2134/**
2135 * Picks the free pages from a chunk.
2136 *
2137 * @returns The new page descriptor table index.
2138 * @param pChunk The chunk.
2139 * @param hGVM The affinity of the chunk. NIL_GVM_HANDLE for no
2140 * affinity.
2141 * @param iPage The current page descriptor table index.
2142 * @param cPages The total number of pages to allocate.
2143 * @param paPages The page descriptor table (input + ouput).
2144 */
2145static uint32_t gmmR0AllocatePagesFromChunk(PGMMCHUNK pChunk, uint16_t const hGVM, uint32_t iPage, uint32_t cPages,
2146 PGMMPAGEDESC paPages)
2147{
2148 PGMMCHUNKFREESET pSet = pChunk->pSet; Assert(pSet);
2149 gmmR0UnlinkChunk(pChunk);
2150
2151 for (; pChunk->cFree && iPage < cPages; iPage++)
2152 gmmR0AllocatePage(pChunk, hGVM, &paPages[iPage]);
2153
2154 gmmR0LinkChunk(pChunk, pSet);
2155 return iPage;
2156}
2157
2158
2159/**
2160 * Registers a new chunk of memory.
2161 *
2162 * This is called by gmmR0AllocateOneChunk and GMMR0AllocateLargePage.
2163 *
2164 * In the GMMR0AllocateLargePage case the GMM_CHUNK_FLAGS_LARGE_PAGE flag is
2165 * set and the chunk will be registered as fully allocated to save time.
2166 *
2167 * @returns VBox status code. On success, the giant GMM lock will be held, the
2168 * caller must release it (ugly).
2169 * @param pGMM Pointer to the GMM instance.
2170 * @param pSet Pointer to the set.
2171 * @param hMemObj The memory object for the chunk.
2172 * @param hGVM The affinity of the chunk. NIL_GVM_HANDLE for no
2173 * affinity.
2174 * @param pSession Same as @a hGVM.
2175 * @param fChunkFlags The chunk flags, GMM_CHUNK_FLAGS_XXX.
2176 * @param cPages The number of pages requested. Zero for large pages.
2177 * @param paPages The page descriptor table (input + output). NULL for
2178 * large pages.
2179 * @param piPage The pointer to the page descriptor table index variable.
2180 * This will be updated. NULL for large pages.
2181 * @param ppChunk Chunk address (out).
2182 *
2183 * @remarks The caller must not own the giant GMM mutex.
2184 * The giant GMM mutex will be acquired and returned acquired in
2185 * the success path. On failure, no locks will be held.
2186 */
2187static int gmmR0RegisterChunk(PGMM pGMM, PGMMCHUNKFREESET pSet, RTR0MEMOBJ hMemObj, uint16_t hGVM, PSUPDRVSESSION pSession,
2188 uint16_t fChunkFlags, uint32_t cPages, PGMMPAGEDESC paPages, uint32_t *piPage, PGMMCHUNK *ppChunk)
2189{
2190 /*
2191 * Validate input & state.
2192 */
2193 Assert(pGMM->hMtxOwner != RTThreadNativeSelf());
2194 Assert(hGVM != NIL_GVM_HANDLE || pGMM->fBoundMemoryMode);
2195 Assert(fChunkFlags == 0 || fChunkFlags == GMM_CHUNK_FLAGS_LARGE_PAGE);
2196 if (!(fChunkFlags &= GMM_CHUNK_FLAGS_LARGE_PAGE))
2197 {
2198 AssertPtr(paPages);
2199 AssertPtr(piPage);
2200 Assert(cPages > 0);
2201 Assert(cPages > *piPage);
2202 }
2203 else
2204 {
2205 Assert(cPages == 0);
2206 Assert(!paPages);
2207 Assert(!piPage);
2208 }
2209
2210#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2211 /*
2212 * Get a ring-0 mapping of the object.
2213 */
2214 uint8_t *pbMapping = (uint8_t *)RTR0MemObjAddress(hMemObj);
2215 if (!pbMapping)
2216 {
2217 RTR0MEMOBJ hMapObj;
2218 int rc = RTR0MemObjMapKernel(&hMapObj, hMemObj, (void *)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
2219 if (RT_SUCCESS(rc))
2220 pbMapping = (uint8_t *)RTR0MemObjAddress(hMapObj);
2221 else
2222 return rc;
2223 AssertPtr(pbMapping);
2224 }
2225#endif
2226
2227 /*
2228 * Allocate a chunk and an ID for it.
2229 */
2230 int rc;
2231 PGMMCHUNK pChunk = (PGMMCHUNK)RTMemAllocZ(sizeof(*pChunk));
2232 if (pChunk)
2233 {
2234 pChunk->Core.Key = gmmR0AllocateChunkId(pGMM);
2235 if ( pChunk->Core.Key != NIL_GMM_CHUNKID
2236 && pChunk->Core.Key <= GMM_CHUNKID_LAST)
2237 {
2238 /*
2239 * Initialize it.
2240 */
2241 pChunk->hMemObj = hMemObj;
2242#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2243 pChunk->pbMapping = pbMapping;
2244#endif
2245 pChunk->hGVM = hGVM;
2246 pChunk->idNumaNode = gmmR0GetCurrentNumaNodeId();
2247 pChunk->iChunkMtx = UINT8_MAX;
2248 pChunk->fFlags = fChunkFlags;
2249 pChunk->uidOwner = pSession ? SUPR0GetSessionUid(pSession) : NIL_RTUID;
2250 /*pChunk->cShared = 0; */
2251
2252 uint32_t const iDstPageFirst = piPage ? *piPage : cPages;
2253 if (!(fChunkFlags & GMM_CHUNK_FLAGS_LARGE_PAGE))
2254 {
2255 /*
2256 * Allocate the requested number of pages from the start of the chunk,
2257 * queue the rest (if any) on the free list.
2258 */
2259 uint32_t const cPagesAlloc = RT_MIN(cPages - iDstPageFirst, GMM_CHUNK_NUM_PAGES);
2260 pChunk->cPrivate = cPagesAlloc;
2261 pChunk->cFree = GMM_CHUNK_NUM_PAGES - cPagesAlloc;
2262 pChunk->iFreeHead = GMM_CHUNK_NUM_PAGES > cPagesAlloc ? cPagesAlloc : UINT16_MAX;
2263
2264 /* Alloc pages: */
2265 uint32_t const idPageChunk = pChunk->Core.Key << GMM_CHUNKID_SHIFT;
2266 uint32_t iDstPage = iDstPageFirst;
2267 uint32_t iPage;
2268 for (iPage = 0; iPage < cPagesAlloc; iPage++, iDstPage++)
2269 {
2270 if (paPages[iDstPage].HCPhysGCPhys <= GMM_GCPHYS_LAST)
2271 pChunk->aPages[iPage].Private.pfn = paPages[iDstPage].HCPhysGCPhys >> GUEST_PAGE_SHIFT;
2272 else
2273 pChunk->aPages[iPage].Private.pfn = GMM_PAGE_PFN_UNSHAREABLE; /* unshareable / unassigned - same thing. */
2274 pChunk->aPages[iPage].Private.hGVM = hGVM;
2275 pChunk->aPages[iPage].Private.u2State = GMM_PAGE_STATE_PRIVATE;
2276
2277 paPages[iDstPage].HCPhysGCPhys = RTR0MemObjGetPagePhysAddr(hMemObj, iPage);
2278 paPages[iDstPage].fZeroed = true;
2279 paPages[iDstPage].idPage = idPageChunk | iPage;
2280 paPages[iDstPage].idSharedPage = NIL_GMM_PAGEID;
2281 }
2282 *piPage = iDstPage;
2283
2284 /* Build free list: */
2285 if (iPage < RT_ELEMENTS(pChunk->aPages))
2286 {
2287 Assert(pChunk->iFreeHead == iPage);
2288 for (; iPage < RT_ELEMENTS(pChunk->aPages) - 1; iPage++)
2289 {
2290 pChunk->aPages[iPage].Free.u2State = GMM_PAGE_STATE_FREE;
2291 pChunk->aPages[iPage].Free.fZeroed = true;
2292 pChunk->aPages[iPage].Free.iNext = iPage + 1;
2293 }
2294 pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.u2State = GMM_PAGE_STATE_FREE;
2295 pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.fZeroed = true;
2296 pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.iNext = UINT16_MAX;
2297 }
2298 else
2299 Assert(pChunk->iFreeHead == UINT16_MAX);
2300 }
2301 else
2302 {
2303 /*
2304 * Large page: Mark all pages as privately allocated (watered down gmmR0AllocatePage).
2305 */
2306 pChunk->cFree = 0;
2307 pChunk->cPrivate = GMM_CHUNK_NUM_PAGES;
2308 pChunk->iFreeHead = UINT16_MAX;
2309
2310 for (unsigned iPage = 0; iPage < RT_ELEMENTS(pChunk->aPages); iPage++)
2311 {
2312 pChunk->aPages[iPage].Private.pfn = GMM_PAGE_PFN_UNSHAREABLE;
2313 pChunk->aPages[iPage].Private.hGVM = hGVM;
2314 pChunk->aPages[iPage].Private.u2State = GMM_PAGE_STATE_PRIVATE;
2315 }
2316 }
2317
2318 /*
2319 * Zero the memory if it wasn't zeroed by the host already.
2320 * This simplifies keeping secret kernel bits from userland and brings
2321 * everyone to the same level wrt allocation zeroing.
2322 */
2323 rc = VINF_SUCCESS;
2324 if (!RTR0MemObjWasZeroInitialized(hMemObj))
2325 {
2326#ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2327 if (!(fChunkFlags & GMM_CHUNK_FLAGS_LARGE_PAGE))
2328 {
2329 for (uint32_t iPage = 0; iPage < GMM_CHUNK_SIZE / HOST_PAGE_SIZE; iPage++)
2330 {
2331 void *pvPage = NULL;
2332 rc = SUPR0HCPhysToVirt(RTR0MemObjGetPagePhysAddr(hMemObj, iPage), &pvPage);
2333 AssertRCBreak(rc);
2334 RT_BZERO(pvPage, HOST_PAGE_SIZE);
2335 }
2336 }
2337 else
2338 {
2339 /* Can do the whole large page in one go. */
2340 void *pvPage = NULL;
2341 rc = SUPR0HCPhysToVirt(RTR0MemObjGetPagePhysAddr(hMemObj, 0), &pvPage);
2342 AssertRC(rc);
2343 if (RT_SUCCESS(rc))
2344 RT_BZERO(pvPage, GMM_CHUNK_SIZE);
2345 }
2346#else
2347 RT_BZERO(pbMapping, GMM_CHUNK_SIZE);
2348#endif
2349 }
2350 if (RT_SUCCESS(rc))
2351 {
2352 *ppChunk = pChunk;
2353
2354 /*
2355 * Allocate a Chunk ID and insert it into the tree.
2356 * This has to be done behind the mutex of course.
2357 */
2358 rc = gmmR0MutexAcquire(pGMM);
2359 if (RT_SUCCESS(rc))
2360 {
2361 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
2362 {
2363 RTSpinlockAcquire(pGMM->hSpinLockTree);
2364 if (RTAvlU32Insert(&pGMM->pChunks, &pChunk->Core))
2365 {
2366 pGMM->cChunks++;
2367 RTListAppend(&pGMM->ChunkList, &pChunk->ListNode);
2368 RTSpinlockRelease(pGMM->hSpinLockTree);
2369
2370 gmmR0LinkChunk(pChunk, pSet);
2371
2372 LogFlow(("gmmR0RegisterChunk: pChunk=%p id=%#x cChunks=%d\n", pChunk, pChunk->Core.Key, pGMM->cChunks));
2373 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
2374 return VINF_SUCCESS;
2375 }
2376
2377 /*
2378 * Bail out.
2379 */
2380 RTSpinlockRelease(pGMM->hSpinLockTree);
2381 rc = VERR_GMM_CHUNK_INSERT;
2382 }
2383 else
2384 rc = VERR_GMM_IS_NOT_SANE;
2385 gmmR0MutexRelease(pGMM);
2386 }
2387 *ppChunk = NULL;
2388 }
2389
2390 /* Undo any page allocations. */
2391 if (!(fChunkFlags & GMM_CHUNK_FLAGS_LARGE_PAGE))
2392 {
2393 uint32_t const cToFree = pChunk->cPrivate;
2394 Assert(*piPage - iDstPageFirst == cToFree);
2395 for (uint32_t iDstPage = iDstPageFirst, iPage = 0; iPage < cToFree; iPage++, iDstPage++)
2396 {
2397 paPages[iDstPageFirst].fZeroed = false;
2398 if (pChunk->aPages[iPage].Private.pfn == GMM_PAGE_PFN_UNSHAREABLE)
2399 paPages[iDstPageFirst].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS;
2400 else
2401 paPages[iDstPageFirst].HCPhysGCPhys = (RTHCPHYS)pChunk->aPages[iPage].Private.pfn << GUEST_PAGE_SHIFT;
2402 paPages[iDstPageFirst].idPage = NIL_GMM_PAGEID;
2403 paPages[iDstPageFirst].idSharedPage = NIL_GMM_PAGEID;
2404 }
2405 *piPage = iDstPageFirst;
2406 }
2407
2408 gmmR0FreeChunkId(pGMM, pChunk->Core.Key);
2409 }
2410 else
2411 rc = VERR_GMM_CHUNK_INSERT;
2412 RTMemFree(pChunk);
2413 }
2414 else
2415 rc = VERR_NO_MEMORY;
2416 return rc;
2417}
2418
2419
2420/**
2421 * Allocate a new chunk, immediately pick the requested pages from it, and adds
2422 * what's remaining to the specified free set.
2423 *
2424 * @note This will leave the giant mutex while allocating the new chunk!
2425 *
2426 * @returns VBox status code.
2427 * @param pGMM Pointer to the GMM instance data.
2428 * @param pGVM Pointer to the kernel-only VM instace data.
2429 * @param pSet Pointer to the free set.
2430 * @param cPages The number of pages requested.
2431 * @param paPages The page descriptor table (input + output).
2432 * @param piPage The pointer to the page descriptor table index variable.
2433 * This will be updated.
2434 */
2435static int gmmR0AllocateChunkNew(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet, uint32_t cPages,
2436 PGMMPAGEDESC paPages, uint32_t *piPage)
2437{
2438 gmmR0MutexRelease(pGMM);
2439
2440 RTR0MEMOBJ hMemObj;
2441 int rc;
2442#ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2443 if (pGMM->fHasWorkingAllocPhysNC)
2444 rc = RTR0MemObjAllocPhysNC(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS);
2445 else
2446#endif
2447 rc = RTR0MemObjAllocPage(&hMemObj, GMM_CHUNK_SIZE, false /*fExecutable*/);
2448 if (RT_SUCCESS(rc))
2449 {
2450 PGMMCHUNK pIgnored;
2451 rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, pGVM->pSession, 0 /*fChunkFlags*/,
2452 cPages, paPages, piPage, &pIgnored);
2453 if (RT_SUCCESS(rc))
2454 return VINF_SUCCESS;
2455
2456 /* bail out */
2457 RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
2458 }
2459
2460 int rc2 = gmmR0MutexAcquire(pGMM);
2461 AssertRCReturn(rc2, RT_FAILURE(rc) ? rc : rc2);
2462 return rc;
2463
2464}
2465
2466
2467/**
2468 * As a last restort we'll pick any page we can get.
2469 *
2470 * @returns The new page descriptor table index.
2471 * @param pSet The set to pick from.
2472 * @param pGVM Pointer to the global VM structure.
2473 * @param uidSelf The UID of the caller.
2474 * @param iPage The current page descriptor table index.
2475 * @param cPages The total number of pages to allocate.
2476 * @param paPages The page descriptor table (input + ouput).
2477 */
2478static uint32_t gmmR0AllocatePagesIndiscriminately(PGMMCHUNKFREESET pSet, PGVM pGVM, RTUID uidSelf,
2479 uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
2480{
2481 unsigned iList = RT_ELEMENTS(pSet->apLists);
2482 while (iList-- > 0)
2483 {
2484 PGMMCHUNK pChunk = pSet->apLists[iList];
2485 while (pChunk)
2486 {
2487 PGMMCHUNK pNext = pChunk->pFreeNext;
2488 if ( pChunk->uidOwner == uidSelf
2489 || ( pChunk->cMappingsX == 0
2490 && pChunk->cFree == (GMM_CHUNK_SIZE >> GUEST_PAGE_SHIFT)))
2491 {
2492 iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
2493 if (iPage >= cPages)
2494 return iPage;
2495 }
2496
2497 pChunk = pNext;
2498 }
2499 }
2500 return iPage;
2501}
2502
2503
2504/**
2505 * Pick pages from empty chunks on the same NUMA node.
2506 *
2507 * @returns The new page descriptor table index.
2508 * @param pSet The set to pick from.
2509 * @param pGVM Pointer to the global VM structure.
2510 * @param uidSelf The UID of the caller.
2511 * @param iPage The current page descriptor table index.
2512 * @param cPages The total number of pages to allocate.
2513 * @param paPages The page descriptor table (input + ouput).
2514 */
2515static uint32_t gmmR0AllocatePagesFromEmptyChunksOnSameNode(PGMMCHUNKFREESET pSet, PGVM pGVM, RTUID uidSelf,
2516 uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
2517{
2518 PGMMCHUNK pChunk = pSet->apLists[GMM_CHUNK_FREE_SET_UNUSED_LIST];
2519 if (pChunk)
2520 {
2521 uint16_t const idNumaNode = gmmR0GetCurrentNumaNodeId();
2522 while (pChunk)
2523 {
2524 PGMMCHUNK pNext = pChunk->pFreeNext;
2525
2526 if ( pChunk->idNumaNode == idNumaNode
2527 && ( pChunk->uidOwner == uidSelf
2528 || pChunk->cMappingsX == 0))
2529 {
2530 pChunk->hGVM = pGVM->hSelf;
2531 pChunk->uidOwner = uidSelf;
2532 iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
2533 if (iPage >= cPages)
2534 {
2535 pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID;
2536 return iPage;
2537 }
2538 }
2539
2540 pChunk = pNext;
2541 }
2542 }
2543 return iPage;
2544}
2545
2546
2547/**
2548 * Pick pages from non-empty chunks on the same NUMA node.
2549 *
2550 * @returns The new page descriptor table index.
2551 * @param pSet The set to pick from.
2552 * @param pGVM Pointer to the global VM structure.
2553 * @param uidSelf The UID of the caller.
2554 * @param iPage The current page descriptor table index.
2555 * @param cPages The total number of pages to allocate.
2556 * @param paPages The page descriptor table (input + ouput).
2557 */
2558static uint32_t gmmR0AllocatePagesFromSameNode(PGMMCHUNKFREESET pSet, PGVM pGVM, RTUID const uidSelf,
2559 uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
2560{
2561 /** @todo start by picking from chunks with about the right size first? */
2562 uint16_t const idNumaNode = gmmR0GetCurrentNumaNodeId();
2563 unsigned iList = GMM_CHUNK_FREE_SET_UNUSED_LIST;
2564 while (iList-- > 0)
2565 {
2566 PGMMCHUNK pChunk = pSet->apLists[iList];
2567 while (pChunk)
2568 {
2569 PGMMCHUNK pNext = pChunk->pFreeNext;
2570
2571 if ( pChunk->idNumaNode == idNumaNode
2572 && pChunk->uidOwner == uidSelf)
2573 {
2574 iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
2575 if (iPage >= cPages)
2576 {
2577 pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID;
2578 return iPage;
2579 }
2580 }
2581
2582 pChunk = pNext;
2583 }
2584 }
2585 return iPage;
2586}
2587
2588
2589/**
2590 * Pick pages that are in chunks already associated with the VM.
2591 *
2592 * @returns The new page descriptor table index.
2593 * @param pGMM Pointer to the GMM instance data.
2594 * @param pGVM Pointer to the global VM structure.
2595 * @param pSet The set to pick from.
2596 * @param iPage The current page descriptor table index.
2597 * @param cPages The total number of pages to allocate.
2598 * @param paPages The page descriptor table (input + ouput).
2599 */
2600static uint32_t gmmR0AllocatePagesAssociatedWithVM(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet,
2601 uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
2602{
2603 uint16_t const hGVM = pGVM->hSelf;
2604
2605 /* Hint. */
2606 if (pGVM->gmm.s.idLastChunkHint != NIL_GMM_CHUNKID)
2607 {
2608 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pGVM->gmm.s.idLastChunkHint);
2609 if (pChunk && pChunk->cFree)
2610 {
2611 iPage = gmmR0AllocatePagesFromChunk(pChunk, hGVM, iPage, cPages, paPages);
2612 if (iPage >= cPages)
2613 return iPage;
2614 }
2615 }
2616
2617 /* Scan. */
2618 for (unsigned iList = 0; iList < RT_ELEMENTS(pSet->apLists); iList++)
2619 {
2620 PGMMCHUNK pChunk = pSet->apLists[iList];
2621 while (pChunk)
2622 {
2623 PGMMCHUNK pNext = pChunk->pFreeNext;
2624
2625 if (pChunk->hGVM == hGVM)
2626 {
2627 iPage = gmmR0AllocatePagesFromChunk(pChunk, hGVM, iPage, cPages, paPages);
2628 if (iPage >= cPages)
2629 {
2630 pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID;
2631 return iPage;
2632 }
2633 }
2634
2635 pChunk = pNext;
2636 }
2637 }
2638 return iPage;
2639}
2640
2641
2642
2643/**
2644 * Pick pages in bound memory mode.
2645 *
2646 * @returns The new page descriptor table index.
2647 * @param pGVM Pointer to the global VM structure.
2648 * @param iPage The current page descriptor table index.
2649 * @param cPages The total number of pages to allocate.
2650 * @param paPages The page descriptor table (input + ouput).
2651 */
2652static uint32_t gmmR0AllocatePagesInBoundMode(PGVM pGVM, uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages)
2653{
2654 for (unsigned iList = 0; iList < RT_ELEMENTS(pGVM->gmm.s.Private.apLists); iList++)
2655 {
2656 PGMMCHUNK pChunk = pGVM->gmm.s.Private.apLists[iList];
2657 while (pChunk)
2658 {
2659 Assert(pChunk->hGVM == pGVM->hSelf);
2660 PGMMCHUNK pNext = pChunk->pFreeNext;
2661 iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages);
2662 if (iPage >= cPages)
2663 return iPage;
2664 pChunk = pNext;
2665 }
2666 }
2667 return iPage;
2668}
2669
2670
2671/**
2672 * Checks if we should start picking pages from chunks of other VMs because
2673 * we're getting close to the system memory or reserved limit.
2674 *
2675 * @returns @c true if we should, @c false if we should first try allocate more
2676 * chunks.
2677 */
2678static bool gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLimits(PGVM pGVM)
2679{
2680 /*
2681 * Don't allocate a new chunk if we're
2682 */
2683 uint64_t cPgReserved = pGVM->gmm.s.Stats.Reserved.cBasePages
2684 + pGVM->gmm.s.Stats.Reserved.cFixedPages
2685 - pGVM->gmm.s.Stats.cBalloonedPages
2686 /** @todo what about shared pages? */;
2687 uint64_t cPgAllocated = pGVM->gmm.s.Stats.Allocated.cBasePages
2688 + pGVM->gmm.s.Stats.Allocated.cFixedPages;
2689 uint64_t cPgDelta = cPgReserved - cPgAllocated;
2690 if (cPgDelta < GMM_CHUNK_NUM_PAGES * 4)
2691 return true;
2692 /** @todo make the threshold configurable, also test the code to see if
2693 * this ever kicks in (we might be reserving too much or smth). */
2694
2695 /*
2696 * Check how close we're to the max memory limit and how many fragments
2697 * there are?...
2698 */
2699 /** @todo */
2700
2701 return false;
2702}
2703
2704
2705/**
2706 * Checks if we should start picking pages from chunks of other VMs because
2707 * there is a lot of free pages around.
2708 *
2709 * @returns @c true if we should, @c false if we should first try allocate more
2710 * chunks.
2711 */
2712static bool gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(PGMM pGMM)
2713{
2714 /*
2715 * Setting the limit at 16 chunks (32 MB) at the moment.
2716 */
2717 if (pGMM->PrivateX.cFreePages >= GMM_CHUNK_NUM_PAGES * 16)
2718 return true;
2719 return false;
2720}
2721
2722
2723/**
2724 * Common worker for GMMR0AllocateHandyPages and GMMR0AllocatePages.
2725 *
2726 * @returns VBox status code:
2727 * @retval VINF_SUCCESS on success.
2728 * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
2729 * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
2730 * that is we're trying to allocate more than we've reserved.
2731 *
2732 * @param pGMM Pointer to the GMM instance data.
2733 * @param pGVM Pointer to the VM.
2734 * @param cPages The number of pages to allocate.
2735 * @param paPages Pointer to the page descriptors. See GMMPAGEDESC for
2736 * details on what is expected on input.
2737 * @param enmAccount The account to charge.
2738 *
2739 * @remarks Caller owns the giant GMM lock.
2740 */
2741static int gmmR0AllocatePagesNew(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMPAGEDESC paPages, GMMACCOUNT enmAccount)
2742{
2743 Assert(pGMM->hMtxOwner == RTThreadNativeSelf());
2744
2745 /*
2746 * Check allocation limits.
2747 */
2748 if (RT_LIKELY(pGMM->cAllocatedPages + cPages <= pGMM->cMaxPages))
2749 { /* likely */ }
2750 else
2751 return VERR_GMM_HIT_GLOBAL_LIMIT;
2752
2753 switch (enmAccount)
2754 {
2755 case GMMACCOUNT_BASE:
2756 if (RT_LIKELY( pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + cPages
2757 <= pGVM->gmm.s.Stats.Reserved.cBasePages))
2758 { /* likely */ }
2759 else
2760 {
2761 Log(("gmmR0AllocatePages:Base: Reserved=%#llx Allocated+Ballooned+Requested=%#llx+%#llx+%#x!\n",
2762 pGVM->gmm.s.Stats.Reserved.cBasePages, pGVM->gmm.s.Stats.Allocated.cBasePages,
2763 pGVM->gmm.s.Stats.cBalloonedPages, cPages));
2764 return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
2765 }
2766 break;
2767 case GMMACCOUNT_SHADOW:
2768 if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cShadowPages + cPages <= pGVM->gmm.s.Stats.Reserved.cShadowPages))
2769 { /* likely */ }
2770 else
2771 {
2772 Log(("gmmR0AllocatePages:Shadow: Reserved=%#x Allocated+Requested=%#x+%#x!\n",
2773 pGVM->gmm.s.Stats.Reserved.cShadowPages, pGVM->gmm.s.Stats.Allocated.cShadowPages, cPages));
2774 return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
2775 }
2776 break;
2777 case GMMACCOUNT_FIXED:
2778 if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cFixedPages + cPages <= pGVM->gmm.s.Stats.Reserved.cFixedPages))
2779 { /* likely */ }
2780 else
2781 {
2782 Log(("gmmR0AllocatePages:Fixed: Reserved=%#x Allocated+Requested=%#x+%#x!\n",
2783 pGVM->gmm.s.Stats.Reserved.cFixedPages, pGVM->gmm.s.Stats.Allocated.cFixedPages, cPages));
2784 return VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
2785 }
2786 break;
2787 default:
2788 AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
2789 }
2790
2791 /*
2792 * Update the accounts before we proceed because we might be leaving the
2793 * protection of the global mutex and thus run the risk of permitting
2794 * too much memory to be allocated.
2795 */
2796 switch (enmAccount)
2797 {
2798 case GMMACCOUNT_BASE: pGVM->gmm.s.Stats.Allocated.cBasePages += cPages; break;
2799 case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages += cPages; break;
2800 case GMMACCOUNT_FIXED: pGVM->gmm.s.Stats.Allocated.cFixedPages += cPages; break;
2801 default: AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
2802 }
2803 pGVM->gmm.s.Stats.cPrivatePages += cPages;
2804 pGMM->cAllocatedPages += cPages;
2805
2806 /*
2807 * Bound mode is also relatively straightforward.
2808 */
2809 uint32_t iPage = 0;
2810 int rc = VINF_SUCCESS;
2811 if (pGMM->fBoundMemoryMode)
2812 {
2813 iPage = gmmR0AllocatePagesInBoundMode(pGVM, iPage, cPages, paPages);
2814 if (iPage < cPages)
2815 do
2816 rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGVM->gmm.s.Private, cPages, paPages, &iPage);
2817 while (iPage < cPages && RT_SUCCESS(rc));
2818 }
2819 /*
2820 * Shared mode is trickier as we should try archive the same locality as
2821 * in bound mode, but smartly make use of non-full chunks allocated by
2822 * other VMs if we're low on memory.
2823 */
2824 else
2825 {
2826 RTUID const uidSelf = SUPR0GetSessionUid(pGVM->pSession);
2827
2828 /* Pick the most optimal pages first. */
2829 iPage = gmmR0AllocatePagesAssociatedWithVM(pGMM, pGVM, &pGMM->PrivateX, iPage, cPages, paPages);
2830 if (iPage < cPages)
2831 {
2832 /* Maybe we should try getting pages from chunks "belonging" to
2833 other VMs before allocating more chunks? */
2834 bool fTriedOnSameAlready = false;
2835 if (gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLimits(pGVM))
2836 {
2837 iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages);
2838 fTriedOnSameAlready = true;
2839 }
2840
2841 /* Allocate memory from empty chunks. */
2842 if (iPage < cPages)
2843 iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages);
2844
2845 /* Grab empty shared chunks. */
2846 if (iPage < cPages)
2847 iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->Shared, pGVM, uidSelf, iPage, cPages, paPages);
2848
2849 /* If there is a lof of free pages spread around, try not waste
2850 system memory on more chunks. (Should trigger defragmentation.) */
2851 if ( !fTriedOnSameAlready
2852 && gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(pGMM))
2853 {
2854 iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages);
2855 if (iPage < cPages)
2856 iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages);
2857 }
2858
2859 /*
2860 * Ok, try allocate new chunks.
2861 */
2862 if (iPage < cPages)
2863 {
2864 do
2865 rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGMM->PrivateX, cPages, paPages, &iPage);
2866 while (iPage < cPages && RT_SUCCESS(rc));
2867
2868#if 0 /* We cannot mix chunks with different UIDs. */
2869 /* If the host is out of memory, take whatever we can get. */
2870 if ( (rc == VERR_NO_MEMORY || rc == VERR_NO_PHYS_MEMORY)
2871 && pGMM->PrivateX.cFreePages + pGMM->Shared.cFreePages >= cPages - iPage)
2872 {
2873 iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, iPage, cPages, paPages);
2874 if (iPage < cPages)
2875 iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->Shared, pGVM, iPage, cPages, paPages);
2876 AssertRelease(iPage == cPages);
2877 rc = VINF_SUCCESS;
2878 }
2879#endif
2880 }
2881 }
2882 }
2883
2884 /*
2885 * Clean up on failure. Since this is bound to be a low-memory condition
2886 * we will give back any empty chunks that might be hanging around.
2887 */
2888 if (RT_SUCCESS(rc))
2889 { /* likely */ }
2890 else
2891 {
2892 /* Update the statistics. */
2893 pGVM->gmm.s.Stats.cPrivatePages -= cPages;
2894 pGMM->cAllocatedPages -= cPages - iPage;
2895 switch (enmAccount)
2896 {
2897 case GMMACCOUNT_BASE: pGVM->gmm.s.Stats.Allocated.cBasePages -= cPages; break;
2898 case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages -= cPages; break;
2899 case GMMACCOUNT_FIXED: pGVM->gmm.s.Stats.Allocated.cFixedPages -= cPages; break;
2900 default: AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
2901 }
2902
2903 /* Release the pages. */
2904 while (iPage-- > 0)
2905 {
2906 uint32_t idPage = paPages[iPage].idPage;
2907 PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
2908 if (RT_LIKELY(pPage))
2909 {
2910 Assert(GMM_PAGE_IS_PRIVATE(pPage));
2911 Assert(pPage->Private.hGVM == pGVM->hSelf);
2912 gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage);
2913 }
2914 else
2915 AssertMsgFailed(("idPage=%#x\n", idPage));
2916
2917 paPages[iPage].idPage = NIL_GMM_PAGEID;
2918 paPages[iPage].idSharedPage = NIL_GMM_PAGEID;
2919 paPages[iPage].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS;
2920 paPages[iPage].fZeroed = false;
2921 }
2922
2923 /* Free empty chunks. */
2924 /** @todo */
2925
2926 /* return the fail status on failure */
2927 return rc;
2928 }
2929 return VINF_SUCCESS;
2930}
2931
2932
2933/**
2934 * Updates the previous allocations and allocates more pages.
2935 *
2936 * The handy pages are always taken from the 'base' memory account.
2937 * The allocated pages are not cleared and will contains random garbage.
2938 *
2939 * @returns VBox status code:
2940 * @retval VINF_SUCCESS on success.
2941 * @retval VERR_NOT_OWNER if the caller is not an EMT.
2942 * @retval VERR_GMM_PAGE_NOT_FOUND if one of the pages to update wasn't found.
2943 * @retval VERR_GMM_PAGE_NOT_PRIVATE if one of the pages to update wasn't a
2944 * private page.
2945 * @retval VERR_GMM_PAGE_NOT_SHARED if one of the pages to update wasn't a
2946 * shared page.
2947 * @retval VERR_GMM_NOT_PAGE_OWNER if one of the pages to be updated wasn't
2948 * owned by the VM.
2949 * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
2950 * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
2951 * that is we're trying to allocate more than we've reserved.
2952 *
2953 * @param pGVM The global (ring-0) VM structure.
2954 * @param idCpu The VCPU id.
2955 * @param cPagesToUpdate The number of pages to update (starting from the head).
2956 * @param cPagesToAlloc The number of pages to allocate (starting from the head).
2957 * @param paPages The array of page descriptors.
2958 * See GMMPAGEDESC for details on what is expected on input.
2959 * @thread EMT(idCpu)
2960 */
2961GMMR0DECL(int) GMMR0AllocateHandyPages(PGVM pGVM, VMCPUID idCpu, uint32_t cPagesToUpdate,
2962 uint32_t cPagesToAlloc, PGMMPAGEDESC paPages)
2963{
2964 LogFlow(("GMMR0AllocateHandyPages: pGVM=%p cPagesToUpdate=%#x cPagesToAlloc=%#x paPages=%p\n",
2965 pGVM, cPagesToUpdate, cPagesToAlloc, paPages));
2966
2967 /*
2968 * Validate & get basics.
2969 * (This is a relatively busy path, so make predictions where possible.)
2970 */
2971 PGMM pGMM;
2972 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
2973 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
2974 if (RT_FAILURE(rc))
2975 return rc;
2976
2977 AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
2978 AssertMsgReturn( (cPagesToUpdate && cPagesToUpdate < 1024)
2979 || (cPagesToAlloc && cPagesToAlloc < 1024),
2980 ("cPagesToUpdate=%#x cPagesToAlloc=%#x\n", cPagesToUpdate, cPagesToAlloc),
2981 VERR_INVALID_PARAMETER);
2982
2983 unsigned iPage = 0;
2984 for (; iPage < cPagesToUpdate; iPage++)
2985 {
2986 AssertMsgReturn( ( paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST
2987 && !(paPages[iPage].HCPhysGCPhys & GUEST_PAGE_OFFSET_MASK))
2988 || paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS
2989 || paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE,
2990 ("#%#x: %RHp\n", iPage, paPages[iPage].HCPhysGCPhys),
2991 VERR_INVALID_PARAMETER);
2992 /* ignore fZeroed here */
2993 AssertMsgReturn( paPages[iPage].idPage <= GMM_PAGEID_LAST
2994 /*|| paPages[iPage].idPage == NIL_GMM_PAGEID*/,
2995 ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
2996 AssertMsgReturn( paPages[iPage].idSharedPage == NIL_GMM_PAGEID
2997 || paPages[iPage].idSharedPage <= GMM_PAGEID_LAST,
2998 ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER);
2999 }
3000
3001 for (; iPage < cPagesToAlloc; iPage++)
3002 {
3003 AssertMsgReturn(paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS, ("#%#x: %RHp\n", iPage, paPages[iPage].HCPhysGCPhys), VERR_INVALID_PARAMETER);
3004 AssertMsgReturn(paPages[iPage].fZeroed == false, ("#%#x: %#x\n", iPage, paPages[iPage].fZeroed), VERR_INVALID_PARAMETER);
3005 AssertMsgReturn(paPages[iPage].idPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
3006 AssertMsgReturn(paPages[iPage].idSharedPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER);
3007 }
3008
3009 /*
3010 * Take the semaphore
3011 */
3012 VMMR0EMTBLOCKCTX Ctx;
3013 PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
3014 rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, "GMMR0AllocateHandyPages", pGMM, &Ctx);
3015 AssertRCReturn(rc, rc);
3016
3017 rc = gmmR0MutexAcquire(pGMM);
3018 if ( RT_SUCCESS(rc)
3019 && GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3020 {
3021 /* No allocations before the initial reservation has been made! */
3022 if (RT_LIKELY( pGVM->gmm.s.Stats.Reserved.cBasePages
3023 && pGVM->gmm.s.Stats.Reserved.cFixedPages
3024 && pGVM->gmm.s.Stats.Reserved.cShadowPages))
3025 {
3026 /*
3027 * Perform the updates.
3028 * Stop on the first error.
3029 */
3030 for (iPage = 0; iPage < cPagesToUpdate; iPage++)
3031 {
3032 if (paPages[iPage].idPage != NIL_GMM_PAGEID)
3033 {
3034 PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idPage);
3035 if (RT_LIKELY(pPage))
3036 {
3037 if (RT_LIKELY(GMM_PAGE_IS_PRIVATE(pPage)))
3038 {
3039 if (RT_LIKELY(pPage->Private.hGVM == pGVM->hSelf))
3040 {
3041 AssertCompile(NIL_RTHCPHYS > GMM_GCPHYS_LAST && GMM_GCPHYS_UNSHAREABLE > GMM_GCPHYS_LAST);
3042 if (RT_LIKELY(paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST))
3043 pPage->Private.pfn = paPages[iPage].HCPhysGCPhys >> GUEST_PAGE_SHIFT;
3044 else if (paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE)
3045 pPage->Private.pfn = GMM_PAGE_PFN_UNSHAREABLE;
3046 /* else: NIL_RTHCPHYS nothing */
3047
3048 paPages[iPage].idPage = NIL_GMM_PAGEID;
3049 paPages[iPage].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS;
3050 paPages[iPage].fZeroed = false;
3051 }
3052 else
3053 {
3054 Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not owner! hGVM=%#x hSelf=%#x\n",
3055 iPage, paPages[iPage].idPage, pPage->Private.hGVM, pGVM->hSelf));
3056 rc = VERR_GMM_NOT_PAGE_OWNER;
3057 break;
3058 }
3059 }
3060 else
3061 {
3062 Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not private! %.*Rhxs (type %d)\n", iPage, paPages[iPage].idPage, sizeof(*pPage), pPage, pPage->Common.u2State));
3063 rc = VERR_GMM_PAGE_NOT_PRIVATE;
3064 break;
3065 }
3066 }
3067 else
3068 {
3069 Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not found! (private)\n", iPage, paPages[iPage].idPage));
3070 rc = VERR_GMM_PAGE_NOT_FOUND;
3071 break;
3072 }
3073 }
3074
3075 if (paPages[iPage].idSharedPage == NIL_GMM_PAGEID)
3076 { /* likely */ }
3077 else
3078 {
3079 PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idSharedPage);
3080 if (RT_LIKELY(pPage))
3081 {
3082 if (RT_LIKELY(GMM_PAGE_IS_SHARED(pPage)))
3083 {
3084 AssertCompile(NIL_RTHCPHYS > GMM_GCPHYS_LAST && GMM_GCPHYS_UNSHAREABLE > GMM_GCPHYS_LAST);
3085 Assert(pPage->Shared.cRefs);
3086 Assert(pGVM->gmm.s.Stats.cSharedPages);
3087 Assert(pGVM->gmm.s.Stats.Allocated.cBasePages);
3088
3089 Log(("GMMR0AllocateHandyPages: free shared page %x cRefs=%d\n", paPages[iPage].idSharedPage, pPage->Shared.cRefs));
3090 pGVM->gmm.s.Stats.cSharedPages--;
3091 pGVM->gmm.s.Stats.Allocated.cBasePages--;
3092 if (!--pPage->Shared.cRefs)
3093 gmmR0FreeSharedPage(pGMM, pGVM, paPages[iPage].idSharedPage, pPage);
3094 else
3095 {
3096 Assert(pGMM->cDuplicatePages);
3097 pGMM->cDuplicatePages--;
3098 }
3099
3100 paPages[iPage].idSharedPage = NIL_GMM_PAGEID;
3101 }
3102 else
3103 {
3104 Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not shared!\n", iPage, paPages[iPage].idSharedPage));
3105 rc = VERR_GMM_PAGE_NOT_SHARED;
3106 break;
3107 }
3108 }
3109 else
3110 {
3111 Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not found! (shared)\n", iPage, paPages[iPage].idSharedPage));
3112 rc = VERR_GMM_PAGE_NOT_FOUND;
3113 break;
3114 }
3115 }
3116 } /* for each page to update */
3117
3118 if (RT_SUCCESS(rc) && cPagesToAlloc > 0)
3119 {
3120#ifdef VBOX_STRICT
3121 for (iPage = 0; iPage < cPagesToAlloc; iPage++)
3122 {
3123 Assert(paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS);
3124 Assert(paPages[iPage].fZeroed == false);
3125 Assert(paPages[iPage].idPage == NIL_GMM_PAGEID);
3126 Assert(paPages[iPage].idSharedPage == NIL_GMM_PAGEID);
3127 }
3128#endif
3129
3130 /*
3131 * Join paths with GMMR0AllocatePages for the allocation.
3132 * Note! gmmR0AllocateMoreChunks may leave the protection of the mutex!
3133 */
3134 rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPagesToAlloc, paPages, GMMACCOUNT_BASE);
3135 }
3136 }
3137 else
3138 rc = VERR_WRONG_ORDER;
3139 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
3140 gmmR0MutexRelease(pGMM);
3141 }
3142 else if (RT_SUCCESS(rc))
3143 {
3144 gmmR0MutexRelease(pGMM);
3145 rc = VERR_GMM_IS_NOT_SANE;
3146 }
3147 VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
3148
3149 LogFlow(("GMMR0AllocateHandyPages: returns %Rrc\n", rc));
3150 return rc;
3151}
3152
3153
3154/**
3155 * Allocate one or more pages.
3156 *
3157 * This is typically used for ROMs and MMIO2 (VRAM) during VM creation.
3158 * The allocated pages are not cleared and will contain random garbage.
3159 *
3160 * @returns VBox status code:
3161 * @retval VINF_SUCCESS on success.
3162 * @retval VERR_NOT_OWNER if the caller is not an EMT.
3163 * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
3164 * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
3165 * that is we're trying to allocate more than we've reserved.
3166 *
3167 * @param pGVM The global (ring-0) VM structure.
3168 * @param idCpu The VCPU id.
3169 * @param cPages The number of pages to allocate.
3170 * @param paPages Pointer to the page descriptors.
3171 * See GMMPAGEDESC for details on what is expected on
3172 * input.
3173 * @param enmAccount The account to charge.
3174 *
3175 * @thread EMT.
3176 */
3177GMMR0DECL(int) GMMR0AllocatePages(PGVM pGVM, VMCPUID idCpu, uint32_t cPages, PGMMPAGEDESC paPages, GMMACCOUNT enmAccount)
3178{
3179 LogFlow(("GMMR0AllocatePages: pGVM=%p cPages=%#x paPages=%p enmAccount=%d\n", pGVM, cPages, paPages, enmAccount));
3180
3181 /*
3182 * Validate, get basics and take the semaphore.
3183 */
3184 PGMM pGMM;
3185 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3186 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
3187 if (RT_FAILURE(rc))
3188 return rc;
3189
3190 AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
3191 AssertMsgReturn(enmAccount > GMMACCOUNT_INVALID && enmAccount < GMMACCOUNT_END, ("%d\n", enmAccount), VERR_INVALID_PARAMETER);
3192 AssertMsgReturn(cPages > 0 && cPages < RT_BIT(32 - GUEST_PAGE_SHIFT), ("%#x\n", cPages), VERR_INVALID_PARAMETER);
3193
3194 for (unsigned iPage = 0; iPage < cPages; iPage++)
3195 {
3196 AssertMsgReturn( paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS
3197 || paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE
3198 || ( enmAccount == GMMACCOUNT_BASE
3199 && paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST
3200 && !(paPages[iPage].HCPhysGCPhys & GUEST_PAGE_OFFSET_MASK)),
3201 ("#%#x: %RHp enmAccount=%d\n", iPage, paPages[iPage].HCPhysGCPhys, enmAccount),
3202 VERR_INVALID_PARAMETER);
3203 AssertMsgReturn(paPages[iPage].fZeroed == false, ("#%#x: %#x\n", iPage, paPages[iPage].fZeroed), VERR_INVALID_PARAMETER);
3204 AssertMsgReturn(paPages[iPage].idPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
3205 AssertMsgReturn(paPages[iPage].idSharedPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER);
3206 }
3207
3208 /*
3209 * Grab the giant mutex and get working.
3210 */
3211 gmmR0MutexAcquire(pGMM);
3212 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3213 {
3214
3215 /* No allocations before the initial reservation has been made! */
3216 if (RT_LIKELY( pGVM->gmm.s.Stats.Reserved.cBasePages
3217 && pGVM->gmm.s.Stats.Reserved.cFixedPages
3218 && pGVM->gmm.s.Stats.Reserved.cShadowPages))
3219 rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPages, paPages, enmAccount);
3220 else
3221 rc = VERR_WRONG_ORDER;
3222 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
3223 }
3224 else
3225 rc = VERR_GMM_IS_NOT_SANE;
3226 gmmR0MutexRelease(pGMM);
3227
3228 LogFlow(("GMMR0AllocatePages: returns %Rrc\n", rc));
3229 return rc;
3230}
3231
3232
3233/**
3234 * VMMR0 request wrapper for GMMR0AllocatePages.
3235 *
3236 * @returns see GMMR0AllocatePages.
3237 * @param pGVM The global (ring-0) VM structure.
3238 * @param idCpu The VCPU id.
3239 * @param pReq Pointer to the request packet.
3240 */
3241GMMR0DECL(int) GMMR0AllocatePagesReq(PGVM pGVM, VMCPUID idCpu, PGMMALLOCATEPAGESREQ pReq)
3242{
3243 /*
3244 * Validate input and pass it on.
3245 */
3246 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
3247 AssertMsgReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF(GMMALLOCATEPAGESREQ, aPages[0]),
3248 ("%#x < %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF(GMMALLOCATEPAGESREQ, aPages[0])),
3249 VERR_INVALID_PARAMETER);
3250 AssertMsgReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMALLOCATEPAGESREQ, aPages[pReq->cPages]),
3251 ("%#x != %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF_DYN(GMMALLOCATEPAGESREQ, aPages[pReq->cPages])),
3252 VERR_INVALID_PARAMETER);
3253
3254 return GMMR0AllocatePages(pGVM, idCpu, pReq->cPages, &pReq->aPages[0], pReq->enmAccount);
3255}
3256
3257
3258/**
3259 * Allocate a large page to represent guest RAM
3260 *
3261 * The allocated pages are zeroed upon return.
3262 *
3263 * @returns VBox status code:
3264 * @retval VINF_SUCCESS on success.
3265 * @retval VERR_NOT_OWNER if the caller is not an EMT.
3266 * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages.
3267 * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit,
3268 * that is we're trying to allocate more than we've reserved.
3269 * @retval VERR_TRY_AGAIN if the host is temporarily out of large pages.
3270 * @returns see GMMR0AllocatePages.
3271 *
3272 * @param pGVM The global (ring-0) VM structure.
3273 * @param idCpu The VCPU id.
3274 * @param cbPage Large page size.
3275 * @param pIdPage Where to return the GMM page ID of the page.
3276 * @param pHCPhys Where to return the host physical address of the page.
3277 */
3278GMMR0DECL(int) GMMR0AllocateLargePage(PGVM pGVM, VMCPUID idCpu, uint32_t cbPage, uint32_t *pIdPage, RTHCPHYS *pHCPhys)
3279{
3280 LogFlow(("GMMR0AllocateLargePage: pGVM=%p cbPage=%x\n", pGVM, cbPage));
3281
3282 AssertPtrReturn(pIdPage, VERR_INVALID_PARAMETER);
3283 *pIdPage = NIL_GMM_PAGEID;
3284 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
3285 *pHCPhys = NIL_RTHCPHYS;
3286 AssertReturn(cbPage == GMM_CHUNK_SIZE, VERR_INVALID_PARAMETER);
3287
3288 /*
3289 * Validate GVM + idCpu, get basics and take the semaphore.
3290 */
3291 PGMM pGMM;
3292 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3293 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
3294 AssertRCReturn(rc, rc);
3295
3296 VMMR0EMTBLOCKCTX Ctx;
3297 PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
3298 rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, "GMMR0AllocateLargePage", pGMM, &Ctx);
3299 AssertRCReturn(rc, rc);
3300
3301 rc = gmmR0MutexAcquire(pGMM);
3302 if (RT_SUCCESS(rc))
3303 {
3304 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3305 {
3306 /*
3307 * Check the quota.
3308 */
3309 /** @todo r=bird: Quota checking could be done w/o the giant mutex but using
3310 * a VM specific mutex... */
3311 if (RT_LIKELY( pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + GMM_CHUNK_NUM_PAGES
3312 <= pGVM->gmm.s.Stats.Reserved.cBasePages))
3313 {
3314 /*
3315 * Allocate a new large page chunk.
3316 *
3317 * Note! We leave the giant GMM lock temporarily as the allocation might
3318 * take a long time. gmmR0RegisterChunk will retake it (ugly).
3319 */
3320 AssertCompile(GMM_CHUNK_SIZE == _2M);
3321 gmmR0MutexRelease(pGMM);
3322
3323 RTR0MEMOBJ hMemObj;
3324 rc = RTR0MemObjAllocLarge(&hMemObj, GMM_CHUNK_SIZE, GMM_CHUNK_SIZE, RTMEMOBJ_ALLOC_LARGE_F_FAST);
3325 if (RT_SUCCESS(rc))
3326 {
3327 *pHCPhys = RTR0MemObjGetPagePhysAddr(hMemObj, 0);
3328
3329 /*
3330 * Register the chunk as fully allocated.
3331 * Note! As mentioned above, this will return owning the mutex on success.
3332 */
3333 PGMMCHUNK pChunk = NULL;
3334 PGMMCHUNKFREESET const pSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX;
3335 rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, pGVM->pSession, GMM_CHUNK_FLAGS_LARGE_PAGE,
3336 0 /*cPages*/, NULL /*paPages*/, NULL /*piPage*/, &pChunk);
3337 if (RT_SUCCESS(rc))
3338 {
3339 /*
3340 * The gmmR0RegisterChunk call already marked all pages allocated,
3341 * so we just have to fill in the return values and update stats now.
3342 */
3343 *pIdPage = pChunk->Core.Key << GMM_CHUNKID_SHIFT;
3344
3345 /* Update accounting. */
3346 pGVM->gmm.s.Stats.Allocated.cBasePages += GMM_CHUNK_NUM_PAGES;
3347 pGVM->gmm.s.Stats.cPrivatePages += GMM_CHUNK_NUM_PAGES;
3348 pGMM->cAllocatedPages += GMM_CHUNK_NUM_PAGES;
3349
3350 gmmR0LinkChunk(pChunk, pSet);
3351 gmmR0MutexRelease(pGMM);
3352
3353 VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
3354 LogFlow(("GMMR0AllocateLargePage: returns VINF_SUCCESS\n"));
3355 return VINF_SUCCESS;
3356 }
3357
3358 /*
3359 * Bail out.
3360 */
3361 RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
3362 *pHCPhys = NIL_RTHCPHYS;
3363 }
3364 /** @todo r=bird: Turn VERR_NO_MEMORY etc into VERR_TRY_AGAIN? Docs say we
3365 * return it, but I am sure IPRT doesn't... */
3366 }
3367 else
3368 {
3369 Log(("GMMR0AllocateLargePage: Reserved=%#llx Allocated+Requested=%#llx+%#x!\n",
3370 pGVM->gmm.s.Stats.Reserved.cBasePages, pGVM->gmm.s.Stats.Allocated.cBasePages, GMM_CHUNK_NUM_PAGES));
3371 gmmR0MutexRelease(pGMM);
3372 rc = VERR_GMM_HIT_VM_ACCOUNT_LIMIT;
3373 }
3374 }
3375 else
3376 {
3377 gmmR0MutexRelease(pGMM);
3378 rc = VERR_GMM_IS_NOT_SANE;
3379 }
3380 }
3381
3382 VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
3383 LogFlow(("GMMR0AllocateLargePage: returns %Rrc\n", rc));
3384 return rc;
3385}
3386
3387
3388/**
3389 * Free a large page.
3390 *
3391 * @returns VBox status code:
3392 * @param pGVM The global (ring-0) VM structure.
3393 * @param idCpu The VCPU id.
3394 * @param idPage The large page id.
3395 */
3396GMMR0DECL(int) GMMR0FreeLargePage(PGVM pGVM, VMCPUID idCpu, uint32_t idPage)
3397{
3398 LogFlow(("GMMR0FreeLargePage: pGVM=%p idPage=%x\n", pGVM, idPage));
3399
3400 /*
3401 * Validate, get basics and take the semaphore.
3402 */
3403 PGMM pGMM;
3404 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3405 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
3406 if (RT_FAILURE(rc))
3407 return rc;
3408
3409 gmmR0MutexAcquire(pGMM);
3410 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3411 {
3412 const unsigned cPages = GMM_CHUNK_NUM_PAGES;
3413
3414 if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages < cPages))
3415 {
3416 Log(("GMMR0FreeLargePage: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cBasePages, cPages));
3417 gmmR0MutexRelease(pGMM);
3418 return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
3419 }
3420
3421 PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
3422 if (RT_LIKELY( pPage
3423 && GMM_PAGE_IS_PRIVATE(pPage)))
3424 {
3425 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
3426 Assert(pChunk);
3427 Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
3428 Assert(pChunk->cPrivate > 0);
3429
3430 /* Release the memory immediately. */
3431 gmmR0FreeChunk(pGMM, NULL, pChunk, false /*fRelaxedSem*/); /** @todo this can be relaxed too! */
3432
3433 /* Update accounting. */
3434 pGVM->gmm.s.Stats.Allocated.cBasePages -= cPages;
3435 pGVM->gmm.s.Stats.cPrivatePages -= cPages;
3436 pGMM->cAllocatedPages -= cPages;
3437 }
3438 else
3439 rc = VERR_GMM_PAGE_NOT_FOUND;
3440 }
3441 else
3442 rc = VERR_GMM_IS_NOT_SANE;
3443
3444 gmmR0MutexRelease(pGMM);
3445 LogFlow(("GMMR0FreeLargePage: returns %Rrc\n", rc));
3446 return rc;
3447}
3448
3449
3450/**
3451 * VMMR0 request wrapper for GMMR0FreeLargePage.
3452 *
3453 * @returns see GMMR0FreeLargePage.
3454 * @param pGVM The global (ring-0) VM structure.
3455 * @param idCpu The VCPU id.
3456 * @param pReq Pointer to the request packet.
3457 */
3458GMMR0DECL(int) GMMR0FreeLargePageReq(PGVM pGVM, VMCPUID idCpu, PGMMFREELARGEPAGEREQ pReq)
3459{
3460 /*
3461 * Validate input and pass it on.
3462 */
3463 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
3464 AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMFREEPAGESREQ),
3465 ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(GMMFREEPAGESREQ)),
3466 VERR_INVALID_PARAMETER);
3467
3468 return GMMR0FreeLargePage(pGVM, idCpu, pReq->idPage);
3469}
3470
3471
3472/**
3473 * @callback_method_impl{FNGVMMR0ENUMCALLBACK,
3474 * Used by gmmR0FreeChunkFlushPerVmTlbs().}
3475 */
3476static DECLCALLBACK(int) gmmR0InvalidatePerVmChunkTlbCallback(PGVM pGVM, void *pvUser)
3477{
3478 RT_NOREF(pvUser);
3479 if (pGVM->gmm.s.hChunkTlbSpinLock != NIL_RTSPINLOCK)
3480 {
3481 RTSpinlockAcquire(pGVM->gmm.s.hChunkTlbSpinLock);
3482 uintptr_t i = RT_ELEMENTS(pGVM->gmm.s.aChunkTlbEntries);
3483 while (i-- > 0)
3484 {
3485 pGVM->gmm.s.aChunkTlbEntries[i].idGeneration = UINT64_MAX;
3486 pGVM->gmm.s.aChunkTlbEntries[i].pChunk = NULL;
3487 }
3488 RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock);
3489 }
3490 return VINF_SUCCESS;
3491}
3492
3493
3494/**
3495 * Called by gmmR0FreeChunk when we reach the threshold for wrapping around the
3496 * free generation ID value.
3497 *
3498 * This is done at 2^62 - 1, which allows us to drop all locks and as it will
3499 * take a while before 12 exa (2 305 843 009 213 693 952) calls to
3500 * gmmR0FreeChunk can be made and causes a real wrap-around. We do two
3501 * invalidation passes and resets the generation ID between then. This will
3502 * make sure there are no false positives.
3503 *
3504 * @param pGMM Pointer to the GMM instance.
3505 */
3506static void gmmR0FreeChunkFlushPerVmTlbs(PGMM pGMM)
3507{
3508 /*
3509 * First invalidation pass.
3510 */
3511 int rc = GVMMR0EnumVMs(gmmR0InvalidatePerVmChunkTlbCallback, NULL);
3512 AssertRCSuccess(rc);
3513
3514 /*
3515 * Reset the generation number.
3516 */
3517 RTSpinlockAcquire(pGMM->hSpinLockTree);
3518 ASMAtomicWriteU64(&pGMM->idFreeGeneration, 1);
3519 RTSpinlockRelease(pGMM->hSpinLockTree);
3520
3521 /*
3522 * Second invalidation pass.
3523 */
3524 rc = GVMMR0EnumVMs(gmmR0InvalidatePerVmChunkTlbCallback, NULL);
3525 AssertRCSuccess(rc);
3526}
3527
3528
3529/**
3530 * Frees a chunk, giving it back to the host OS.
3531 *
3532 * @param pGMM Pointer to the GMM instance.
3533 * @param pGVM This is set when called from GMMR0CleanupVM so we can
3534 * unmap and free the chunk in one go.
3535 * @param pChunk The chunk to free.
3536 * @param fRelaxedSem Whether we can release the semaphore while doing the
3537 * freeing (@c true) or not.
3538 */
3539static bool gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem)
3540{
3541 Assert(pChunk->Core.Key != NIL_GMM_CHUNKID);
3542
3543 GMMR0CHUNKMTXSTATE MtxState;
3544 gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
3545
3546 /*
3547 * Cleanup hack! Unmap the chunk from the callers address space.
3548 * This shouldn't happen, so screw lock contention...
3549 */
3550 if (pChunk->cMappingsX && pGVM)
3551 gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk);
3552
3553 /*
3554 * If there are current mappings of the chunk, then request the
3555 * VMs to unmap them. Reposition the chunk in the free list so
3556 * it won't be a likely candidate for allocations.
3557 */
3558 if (pChunk->cMappingsX)
3559 {
3560 /** @todo R0 -> VM request */
3561 /* The chunk can be mapped by more than one VM if fBoundMemoryMode is false! */
3562 Log(("gmmR0FreeChunk: chunk still has %d mappings; don't free!\n", pChunk->cMappingsX));
3563 gmmR0ChunkMutexRelease(&MtxState, pChunk);
3564 return false;
3565 }
3566
3567
3568 /*
3569 * Save and trash the handle.
3570 */
3571 RTR0MEMOBJ const hMemObj = pChunk->hMemObj;
3572 pChunk->hMemObj = NIL_RTR0MEMOBJ;
3573
3574 /*
3575 * Unlink it from everywhere.
3576 */
3577 gmmR0UnlinkChunk(pChunk);
3578
3579 RTSpinlockAcquire(pGMM->hSpinLockTree);
3580
3581 RTListNodeRemove(&pChunk->ListNode);
3582
3583 PAVLU32NODECORE pCore = RTAvlU32Remove(&pGMM->pChunks, pChunk->Core.Key);
3584 Assert(pCore == &pChunk->Core); NOREF(pCore);
3585
3586 PGMMCHUNKTLBE pTlbe = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(pChunk->Core.Key)];
3587 if (pTlbe->pChunk == pChunk)
3588 {
3589 pTlbe->idChunk = NIL_GMM_CHUNKID;
3590 pTlbe->pChunk = NULL;
3591 }
3592
3593 Assert(pGMM->cChunks > 0);
3594 pGMM->cChunks--;
3595
3596 uint64_t const idFreeGeneration = ASMAtomicIncU64(&pGMM->idFreeGeneration);
3597
3598 RTSpinlockRelease(pGMM->hSpinLockTree);
3599
3600 pGMM->cFreedChunks++;
3601
3602 /* Drop the lock. */
3603 gmmR0ChunkMutexRelease(&MtxState, NULL);
3604 if (fRelaxedSem)
3605 gmmR0MutexRelease(pGMM);
3606
3607 /*
3608 * Flush per VM chunk TLBs if we're getting remotely close to a generation wraparound.
3609 */
3610 if (idFreeGeneration == UINT64_MAX / 4)
3611 gmmR0FreeChunkFlushPerVmTlbs(pGMM);
3612
3613 /*
3614 * Free the Chunk ID and all memory associated with the chunk.
3615 */
3616 gmmR0FreeChunkId(pGMM, pChunk->Core.Key);
3617 pChunk->Core.Key = NIL_GMM_CHUNKID;
3618
3619 RTMemFree(pChunk->paMappingsX);
3620 pChunk->paMappingsX = NULL;
3621
3622 RTMemFree(pChunk);
3623
3624#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
3625 int rc = RTR0MemObjFree(hMemObj, true /* fFreeMappings */);
3626#else
3627 int rc = RTR0MemObjFree(hMemObj, false /* fFreeMappings */);
3628#endif
3629 AssertLogRelRC(rc);
3630
3631 if (fRelaxedSem)
3632 gmmR0MutexAcquire(pGMM);
3633 return fRelaxedSem;
3634}
3635
3636
3637/**
3638 * Free page worker.
3639 *
3640 * The caller does all the statistic decrementing, we do all the incrementing.
3641 *
3642 * @param pGMM Pointer to the GMM instance data.
3643 * @param pGVM Pointer to the GVM instance.
3644 * @param pChunk Pointer to the chunk this page belongs to.
3645 * @param idPage The Page ID.
3646 * @param pPage Pointer to the page.
3647 */
3648static void gmmR0FreePageWorker(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, uint32_t idPage, PGMMPAGE pPage)
3649{
3650 Log3(("F pPage=%p iPage=%#x/%#x u2State=%d iFreeHead=%#x\n",
3651 pPage, pPage - &pChunk->aPages[0], idPage, pPage->Common.u2State, pChunk->iFreeHead)); NOREF(idPage);
3652
3653 /*
3654 * Put the page on the free list.
3655 */
3656 pPage->u = 0;
3657 pPage->Free.u2State = GMM_PAGE_STATE_FREE;
3658 pPage->Free.fZeroed = false;
3659 Assert(pChunk->iFreeHead < RT_ELEMENTS(pChunk->aPages) || pChunk->iFreeHead == UINT16_MAX);
3660 pPage->Free.iNext = pChunk->iFreeHead;
3661 pChunk->iFreeHead = pPage - &pChunk->aPages[0];
3662
3663 /*
3664 * Update statistics (the cShared/cPrivate stats are up to date already),
3665 * and relink the chunk if necessary.
3666 */
3667 unsigned const cFree = pChunk->cFree;
3668 if ( !cFree
3669 || gmmR0SelectFreeSetList(cFree) != gmmR0SelectFreeSetList(cFree + 1))
3670 {
3671 gmmR0UnlinkChunk(pChunk);
3672 pChunk->cFree++;
3673 gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk);
3674 }
3675 else
3676 {
3677 pChunk->cFree = cFree + 1;
3678 pChunk->pSet->cFreePages++;
3679 }
3680
3681 /*
3682 * If the chunk becomes empty, consider giving memory back to the host OS.
3683 *
3684 * The current strategy is to try give it back if there are other chunks
3685 * in this free list, meaning if there are at least 240 free pages in this
3686 * category. Note that since there are probably mappings of the chunk,
3687 * it won't be freed up instantly, which probably screws up this logic
3688 * a bit...
3689 */
3690 /** @todo Do this on the way out. */
3691 if (RT_LIKELY( pChunk->cFree != GMM_CHUNK_NUM_PAGES
3692 || pChunk->pFreeNext == NULL
3693 || pChunk->pFreePrev == NULL /** @todo this is probably misfiring, see reset... */))
3694 { /* likely */ }
3695 else
3696 gmmR0FreeChunk(pGMM, NULL, pChunk, false);
3697}
3698
3699
3700/**
3701 * Frees a shared page, the page is known to exist and be valid and such.
3702 *
3703 * @param pGMM Pointer to the GMM instance.
3704 * @param pGVM Pointer to the GVM instance.
3705 * @param idPage The page id.
3706 * @param pPage The page structure.
3707 */
3708DECLINLINE(void) gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage)
3709{
3710 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
3711 Assert(pChunk);
3712 Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
3713 Assert(pChunk->cShared > 0);
3714 Assert(pGMM->cSharedPages > 0);
3715 Assert(pGMM->cAllocatedPages > 0);
3716 Assert(!pPage->Shared.cRefs);
3717
3718 pChunk->cShared--;
3719 pGMM->cAllocatedPages--;
3720 pGMM->cSharedPages--;
3721 gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage);
3722}
3723
3724
3725/**
3726 * Frees a private page, the page is known to exist and be valid and such.
3727 *
3728 * @param pGMM Pointer to the GMM instance.
3729 * @param pGVM Pointer to the GVM instance.
3730 * @param idPage The page id.
3731 * @param pPage The page structure.
3732 */
3733DECLINLINE(void) gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage)
3734{
3735 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
3736 Assert(pChunk);
3737 Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
3738 Assert(pChunk->cPrivate > 0);
3739 Assert(pGMM->cAllocatedPages > 0);
3740
3741 pChunk->cPrivate--;
3742 pGMM->cAllocatedPages--;
3743 gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage);
3744}
3745
3746
3747/**
3748 * Common worker for GMMR0FreePages and GMMR0BalloonedPages.
3749 *
3750 * @returns VBox status code:
3751 * @retval xxx
3752 *
3753 * @param pGMM Pointer to the GMM instance data.
3754 * @param pGVM Pointer to the VM.
3755 * @param cPages The number of pages to free.
3756 * @param paPages Pointer to the page descriptors.
3757 * @param enmAccount The account this relates to.
3758 */
3759static int gmmR0FreePages(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMFREEPAGEDESC paPages, GMMACCOUNT enmAccount)
3760{
3761 /*
3762 * Check that the request isn't impossible wrt to the account status.
3763 */
3764 switch (enmAccount)
3765 {
3766 case GMMACCOUNT_BASE:
3767 if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages < cPages))
3768 {
3769 Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cBasePages, cPages));
3770 return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
3771 }
3772 break;
3773 case GMMACCOUNT_SHADOW:
3774 if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cShadowPages < cPages))
3775 {
3776 Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cShadowPages, cPages));
3777 return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
3778 }
3779 break;
3780 case GMMACCOUNT_FIXED:
3781 if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cFixedPages < cPages))
3782 {
3783 Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cFixedPages, cPages));
3784 return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
3785 }
3786 break;
3787 default:
3788 AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3789 }
3790
3791 /*
3792 * Walk the descriptors and free the pages.
3793 *
3794 * Statistics (except the account) are being updated as we go along,
3795 * unlike the alloc code. Also, stop on the first error.
3796 */
3797 int rc = VINF_SUCCESS;
3798 uint32_t iPage;
3799 for (iPage = 0; iPage < cPages; iPage++)
3800 {
3801 uint32_t idPage = paPages[iPage].idPage;
3802 PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage);
3803 if (RT_LIKELY(pPage))
3804 {
3805 if (RT_LIKELY(GMM_PAGE_IS_PRIVATE(pPage)))
3806 {
3807 if (RT_LIKELY(pPage->Private.hGVM == pGVM->hSelf))
3808 {
3809 Assert(pGVM->gmm.s.Stats.cPrivatePages);
3810 pGVM->gmm.s.Stats.cPrivatePages--;
3811 gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage);
3812 }
3813 else
3814 {
3815 Log(("gmmR0AllocatePages: #%#x/%#x: not owner! hGVM=%#x hSelf=%#x\n", iPage, idPage,
3816 pPage->Private.hGVM, pGVM->hSelf));
3817 rc = VERR_GMM_NOT_PAGE_OWNER;
3818 break;
3819 }
3820 }
3821 else if (RT_LIKELY(GMM_PAGE_IS_SHARED(pPage)))
3822 {
3823 Assert(pGVM->gmm.s.Stats.cSharedPages);
3824 Assert(pPage->Shared.cRefs);
3825#if defined(VBOX_WITH_PAGE_SHARING) && defined(VBOX_STRICT)
3826 if (pPage->Shared.u14Checksum)
3827 {
3828 uint32_t uChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage);
3829 uChecksum &= UINT32_C(0x00003fff);
3830 AssertMsg(!uChecksum || uChecksum == pPage->Shared.u14Checksum,
3831 ("%#x vs %#x - idPage=%#x\n", uChecksum, pPage->Shared.u14Checksum, idPage));
3832 }
3833#endif
3834 pGVM->gmm.s.Stats.cSharedPages--;
3835 if (!--pPage->Shared.cRefs)
3836 gmmR0FreeSharedPage(pGMM, pGVM, idPage, pPage);
3837 else
3838 {
3839 Assert(pGMM->cDuplicatePages);
3840 pGMM->cDuplicatePages--;
3841 }
3842 }
3843 else
3844 {
3845 Log(("gmmR0AllocatePages: #%#x/%#x: already free!\n", iPage, idPage));
3846 rc = VERR_GMM_PAGE_ALREADY_FREE;
3847 break;
3848 }
3849 }
3850 else
3851 {
3852 Log(("gmmR0AllocatePages: #%#x/%#x: not found!\n", iPage, idPage));
3853 rc = VERR_GMM_PAGE_NOT_FOUND;
3854 break;
3855 }
3856 paPages[iPage].idPage = NIL_GMM_PAGEID;
3857 }
3858
3859 /*
3860 * Update the account.
3861 */
3862 switch (enmAccount)
3863 {
3864 case GMMACCOUNT_BASE: pGVM->gmm.s.Stats.Allocated.cBasePages -= iPage; break;
3865 case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages -= iPage; break;
3866 case GMMACCOUNT_FIXED: pGVM->gmm.s.Stats.Allocated.cFixedPages -= iPage; break;
3867 default:
3868 AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3869 }
3870
3871 /*
3872 * Any threshold stuff to be done here?
3873 */
3874
3875 return rc;
3876}
3877
3878
3879/**
3880 * Free one or more pages.
3881 *
3882 * This is typically used at reset time or power off.
3883 *
3884 * @returns VBox status code:
3885 * @retval xxx
3886 *
3887 * @param pGVM The global (ring-0) VM structure.
3888 * @param idCpu The VCPU id.
3889 * @param cPages The number of pages to allocate.
3890 * @param paPages Pointer to the page descriptors containing the page IDs
3891 * for each page.
3892 * @param enmAccount The account this relates to.
3893 * @thread EMT.
3894 */
3895GMMR0DECL(int) GMMR0FreePages(PGVM pGVM, VMCPUID idCpu, uint32_t cPages, PGMMFREEPAGEDESC paPages, GMMACCOUNT enmAccount)
3896{
3897 LogFlow(("GMMR0FreePages: pGVM=%p cPages=%#x paPages=%p enmAccount=%d\n", pGVM, cPages, paPages, enmAccount));
3898
3899 /*
3900 * Validate input and get the basics.
3901 */
3902 PGMM pGMM;
3903 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3904 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
3905 if (RT_FAILURE(rc))
3906 return rc;
3907
3908 AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
3909 AssertMsgReturn(enmAccount > GMMACCOUNT_INVALID && enmAccount < GMMACCOUNT_END, ("%d\n", enmAccount), VERR_INVALID_PARAMETER);
3910 AssertMsgReturn(cPages > 0 && cPages < RT_BIT(32 - GUEST_PAGE_SHIFT), ("%#x\n", cPages), VERR_INVALID_PARAMETER);
3911
3912 for (unsigned iPage = 0; iPage < cPages; iPage++)
3913 AssertMsgReturn( paPages[iPage].idPage <= GMM_PAGEID_LAST
3914 /*|| paPages[iPage].idPage == NIL_GMM_PAGEID*/,
3915 ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER);
3916
3917 /*
3918 * Take the semaphore and call the worker function.
3919 */
3920 gmmR0MutexAcquire(pGMM);
3921 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
3922 {
3923 rc = gmmR0FreePages(pGMM, pGVM, cPages, paPages, enmAccount);
3924 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
3925 }
3926 else
3927 rc = VERR_GMM_IS_NOT_SANE;
3928 gmmR0MutexRelease(pGMM);
3929 LogFlow(("GMMR0FreePages: returns %Rrc\n", rc));
3930 return rc;
3931}
3932
3933
3934/**
3935 * VMMR0 request wrapper for GMMR0FreePages.
3936 *
3937 * @returns see GMMR0FreePages.
3938 * @param pGVM The global (ring-0) VM structure.
3939 * @param idCpu The VCPU id.
3940 * @param pReq Pointer to the request packet.
3941 */
3942GMMR0DECL(int) GMMR0FreePagesReq(PGVM pGVM, VMCPUID idCpu, PGMMFREEPAGESREQ pReq)
3943{
3944 /*
3945 * Validate input and pass it on.
3946 */
3947 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
3948 AssertMsgReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF(GMMFREEPAGESREQ, aPages[0]),
3949 ("%#x < %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF(GMMFREEPAGESREQ, aPages[0])),
3950 VERR_INVALID_PARAMETER);
3951 AssertMsgReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[pReq->cPages]),
3952 ("%#x != %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[pReq->cPages])),
3953 VERR_INVALID_PARAMETER);
3954
3955 return GMMR0FreePages(pGVM, idCpu, pReq->cPages, &pReq->aPages[0], pReq->enmAccount);
3956}
3957
3958
3959/**
3960 * Report back on a memory ballooning request.
3961 *
3962 * The request may or may not have been initiated by the GMM. If it was initiated
3963 * by the GMM it is important that this function is called even if no pages were
3964 * ballooned.
3965 *
3966 * @returns VBox status code:
3967 * @retval VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH
3968 * @retval VERR_GMM_ATTEMPT_TO_DEFLATE_TOO_MUCH
3969 * @retval VERR_GMM_OVERCOMMITTED_TRY_AGAIN_IN_A_BIT - reset condition
3970 * indicating that we won't necessarily have sufficient RAM to boot
3971 * the VM again and that it should pause until this changes (we'll try
3972 * balloon some other VM). (For standard deflate we have little choice
3973 * but to hope the VM won't use the memory that was returned to it.)
3974 *
3975 * @param pGVM The global (ring-0) VM structure.
3976 * @param idCpu The VCPU id.
3977 * @param enmAction Inflate/deflate/reset.
3978 * @param cBalloonedPages The number of pages that was ballooned.
3979 *
3980 * @thread EMT(idCpu)
3981 */
3982GMMR0DECL(int) GMMR0BalloonedPages(PGVM pGVM, VMCPUID idCpu, GMMBALLOONACTION enmAction, uint32_t cBalloonedPages)
3983{
3984 LogFlow(("GMMR0BalloonedPages: pGVM=%p enmAction=%d cBalloonedPages=%#x\n",
3985 pGVM, enmAction, cBalloonedPages));
3986
3987 AssertMsgReturn(cBalloonedPages < RT_BIT(32 - GUEST_PAGE_SHIFT), ("%#x\n", cBalloonedPages), VERR_INVALID_PARAMETER);
3988
3989 /*
3990 * Validate input and get the basics.
3991 */
3992 PGMM pGMM;
3993 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
3994 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
3995 if (RT_FAILURE(rc))
3996 return rc;
3997
3998 /*
3999 * Take the semaphore and do some more validations.
4000 */
4001 gmmR0MutexAcquire(pGMM);
4002 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
4003 {
4004 switch (enmAction)
4005 {
4006 case GMMBALLOONACTION_INFLATE:
4007 {
4008 if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + cBalloonedPages
4009 <= pGVM->gmm.s.Stats.Reserved.cBasePages))
4010 {
4011 /*
4012 * Record the ballooned memory.
4013 */
4014 pGMM->cBalloonedPages += cBalloonedPages;
4015 if (pGVM->gmm.s.Stats.cReqBalloonedPages)
4016 {
4017 /* Codepath never taken. Might be interesting in the future to request ballooned memory from guests in low memory conditions.. */
4018 AssertFailed();
4019
4020 pGVM->gmm.s.Stats.cBalloonedPages += cBalloonedPages;
4021 pGVM->gmm.s.Stats.cReqActuallyBalloonedPages += cBalloonedPages;
4022 Log(("GMMR0BalloonedPages: +%#x - Global=%#llx / VM: Total=%#llx Req=%#llx Actual=%#llx (pending)\n",
4023 cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages,
4024 pGVM->gmm.s.Stats.cReqBalloonedPages, pGVM->gmm.s.Stats.cReqActuallyBalloonedPages));
4025 }
4026 else
4027 {
4028 pGVM->gmm.s.Stats.cBalloonedPages += cBalloonedPages;
4029 Log(("GMMR0BalloonedPages: +%#x - Global=%#llx / VM: Total=%#llx (user)\n",
4030 cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages));
4031 }
4032 }
4033 else
4034 {
4035 Log(("GMMR0BalloonedPages: cBasePages=%#llx Total=%#llx cBalloonedPages=%#llx Reserved=%#llx\n",
4036 pGVM->gmm.s.Stats.Allocated.cBasePages, pGVM->gmm.s.Stats.cBalloonedPages, cBalloonedPages,
4037 pGVM->gmm.s.Stats.Reserved.cBasePages));
4038 rc = VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH;
4039 }
4040 break;
4041 }
4042
4043 case GMMBALLOONACTION_DEFLATE:
4044 {
4045 /* Deflate. */
4046 if (pGVM->gmm.s.Stats.cBalloonedPages >= cBalloonedPages)
4047 {
4048 /*
4049 * Record the ballooned memory.
4050 */
4051 Assert(pGMM->cBalloonedPages >= cBalloonedPages);
4052 pGMM->cBalloonedPages -= cBalloonedPages;
4053 pGVM->gmm.s.Stats.cBalloonedPages -= cBalloonedPages;
4054 if (pGVM->gmm.s.Stats.cReqDeflatePages)
4055 {
4056 AssertFailed(); /* This is path is for later. */
4057 Log(("GMMR0BalloonedPages: -%#x - Global=%#llx / VM: Total=%#llx Req=%#llx\n",
4058 cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages, pGVM->gmm.s.Stats.cReqDeflatePages));
4059
4060 /*
4061 * Anything we need to do here now when the request has been completed?
4062 */
4063 pGVM->gmm.s.Stats.cReqDeflatePages = 0;
4064 }
4065 else
4066 Log(("GMMR0BalloonedPages: -%#x - Global=%#llx / VM: Total=%#llx (user)\n",
4067 cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages));
4068 }
4069 else
4070 {
4071 Log(("GMMR0BalloonedPages: Total=%#llx cBalloonedPages=%#llx\n", pGVM->gmm.s.Stats.cBalloonedPages, cBalloonedPages));
4072 rc = VERR_GMM_ATTEMPT_TO_DEFLATE_TOO_MUCH;
4073 }
4074 break;
4075 }
4076
4077 case GMMBALLOONACTION_RESET:
4078 {
4079 /* Reset to an empty balloon. */
4080 Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages);
4081
4082 pGMM->cBalloonedPages -= pGVM->gmm.s.Stats.cBalloonedPages;
4083 pGVM->gmm.s.Stats.cBalloonedPages = 0;
4084 break;
4085 }
4086
4087 default:
4088 rc = VERR_INVALID_PARAMETER;
4089 break;
4090 }
4091 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
4092 }
4093 else
4094 rc = VERR_GMM_IS_NOT_SANE;
4095
4096 gmmR0MutexRelease(pGMM);
4097 LogFlow(("GMMR0BalloonedPages: returns %Rrc\n", rc));
4098 return rc;
4099}
4100
4101
4102/**
4103 * VMMR0 request wrapper for GMMR0BalloonedPages.
4104 *
4105 * @returns see GMMR0BalloonedPages.
4106 * @param pGVM The global (ring-0) VM structure.
4107 * @param idCpu The VCPU id.
4108 * @param pReq Pointer to the request packet.
4109 */
4110GMMR0DECL(int) GMMR0BalloonedPagesReq(PGVM pGVM, VMCPUID idCpu, PGMMBALLOONEDPAGESREQ pReq)
4111{
4112 /*
4113 * Validate input and pass it on.
4114 */
4115 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
4116 AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMBALLOONEDPAGESREQ),
4117 ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMBALLOONEDPAGESREQ)),
4118 VERR_INVALID_PARAMETER);
4119
4120 return GMMR0BalloonedPages(pGVM, idCpu, pReq->enmAction, pReq->cBalloonedPages);
4121}
4122
4123
4124/**
4125 * Return memory statistics for the hypervisor
4126 *
4127 * @returns VBox status code.
4128 * @param pReq Pointer to the request packet.
4129 */
4130GMMR0DECL(int) GMMR0QueryHypervisorMemoryStatsReq(PGMMMEMSTATSREQ pReq)
4131{
4132 /*
4133 * Validate input and pass it on.
4134 */
4135 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
4136 AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMMEMSTATSREQ),
4137 ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMMEMSTATSREQ)),
4138 VERR_INVALID_PARAMETER);
4139
4140 /*
4141 * Validate input and get the basics.
4142 */
4143 PGMM pGMM;
4144 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4145 pReq->cAllocPages = pGMM->cAllocatedPages;
4146 pReq->cFreePages = (pGMM->cChunks << (GMM_CHUNK_SHIFT - GUEST_PAGE_SHIFT)) - pGMM->cAllocatedPages;
4147 pReq->cBalloonedPages = pGMM->cBalloonedPages;
4148 pReq->cMaxPages = pGMM->cMaxPages;
4149 pReq->cSharedPages = pGMM->cDuplicatePages;
4150 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
4151
4152 return VINF_SUCCESS;
4153}
4154
4155
4156/**
4157 * Return memory statistics for the VM
4158 *
4159 * @returns VBox status code.
4160 * @param pGVM The global (ring-0) VM structure.
4161 * @param idCpu Cpu id.
4162 * @param pReq Pointer to the request packet.
4163 *
4164 * @thread EMT(idCpu)
4165 */
4166GMMR0DECL(int) GMMR0QueryMemoryStatsReq(PGVM pGVM, VMCPUID idCpu, PGMMMEMSTATSREQ pReq)
4167{
4168 /*
4169 * Validate input and pass it on.
4170 */
4171 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
4172 AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMMEMSTATSREQ),
4173 ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMMEMSTATSREQ)),
4174 VERR_INVALID_PARAMETER);
4175
4176 /*
4177 * Validate input and get the basics.
4178 */
4179 PGMM pGMM;
4180 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4181 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
4182 if (RT_FAILURE(rc))
4183 return rc;
4184
4185 /*
4186 * Take the semaphore and do some more validations.
4187 */
4188 gmmR0MutexAcquire(pGMM);
4189 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
4190 {
4191 pReq->cAllocPages = pGVM->gmm.s.Stats.Allocated.cBasePages;
4192 pReq->cBalloonedPages = pGVM->gmm.s.Stats.cBalloonedPages;
4193 pReq->cMaxPages = pGVM->gmm.s.Stats.Reserved.cBasePages;
4194 pReq->cFreePages = pReq->cMaxPages - pReq->cAllocPages;
4195 }
4196 else
4197 rc = VERR_GMM_IS_NOT_SANE;
4198
4199 gmmR0MutexRelease(pGMM);
4200 LogFlow(("GMMR3QueryVMMemoryStats: returns %Rrc\n", rc));
4201 return rc;
4202}
4203
4204
4205/**
4206 * Worker for gmmR0UnmapChunk and gmmr0FreeChunk.
4207 *
4208 * Don't call this in legacy allocation mode!
4209 *
4210 * @returns VBox status code.
4211 * @param pGMM Pointer to the GMM instance data.
4212 * @param pGVM Pointer to the Global VM structure.
4213 * @param pChunk Pointer to the chunk to be unmapped.
4214 */
4215static int gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk)
4216{
4217 RT_NOREF_PV(pGMM);
4218
4219 /*
4220 * Find the mapping and try unmapping it.
4221 */
4222 uint32_t cMappings = pChunk->cMappingsX;
4223 for (uint32_t i = 0; i < cMappings; i++)
4224 {
4225 Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ);
4226 if (pChunk->paMappingsX[i].pGVM == pGVM)
4227 {
4228 /* unmap */
4229 int rc = RTR0MemObjFree(pChunk->paMappingsX[i].hMapObj, false /* fFreeMappings (NA) */);
4230 if (RT_SUCCESS(rc))
4231 {
4232 /* update the record. */
4233 cMappings--;
4234 if (i < cMappings)
4235 pChunk->paMappingsX[i] = pChunk->paMappingsX[cMappings];
4236 pChunk->paMappingsX[cMappings].hMapObj = NIL_RTR0MEMOBJ;
4237 pChunk->paMappingsX[cMappings].pGVM = NULL;
4238 Assert(pChunk->cMappingsX - 1U == cMappings);
4239 pChunk->cMappingsX = cMappings;
4240 }
4241
4242 return rc;
4243 }
4244 }
4245
4246 Log(("gmmR0UnmapChunk: Chunk %#x is not mapped into pGVM=%p/%#x\n", pChunk->Core.Key, pGVM, pGVM->hSelf));
4247 return VERR_GMM_CHUNK_NOT_MAPPED;
4248}
4249
4250
4251/**
4252 * Unmaps a chunk previously mapped into the address space of the current process.
4253 *
4254 * @returns VBox status code.
4255 * @param pGMM Pointer to the GMM instance data.
4256 * @param pGVM Pointer to the Global VM structure.
4257 * @param pChunk Pointer to the chunk to be unmapped.
4258 * @param fRelaxedSem Whether we can release the semaphore while doing the
4259 * mapping (@c true) or not.
4260 */
4261static int gmmR0UnmapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem)
4262{
4263 /*
4264 * Lock the chunk and if possible leave the giant GMM lock.
4265 */
4266 GMMR0CHUNKMTXSTATE MtxState;
4267 int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk,
4268 fRelaxedSem ? GMMR0CHUNK_MTX_RETAKE_GIANT : GMMR0CHUNK_MTX_KEEP_GIANT);
4269 if (RT_SUCCESS(rc))
4270 {
4271 rc = gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk);
4272 gmmR0ChunkMutexRelease(&MtxState, pChunk);
4273 }
4274 return rc;
4275}
4276
4277
4278/**
4279 * Worker for gmmR0MapChunk.
4280 *
4281 * @returns VBox status code.
4282 * @param pGMM Pointer to the GMM instance data.
4283 * @param pGVM Pointer to the Global VM structure.
4284 * @param pChunk Pointer to the chunk to be mapped.
4285 * @param ppvR3 Where to store the ring-3 address of the mapping.
4286 * In the VERR_GMM_CHUNK_ALREADY_MAPPED case, this will be
4287 * contain the address of the existing mapping.
4288 */
4289static int gmmR0MapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3)
4290{
4291 RT_NOREF(pGMM);
4292
4293 /*
4294 * Check to see if the chunk is already mapped.
4295 */
4296 for (uint32_t i = 0; i < pChunk->cMappingsX; i++)
4297 {
4298 Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ);
4299 if (pChunk->paMappingsX[i].pGVM == pGVM)
4300 {
4301 *ppvR3 = RTR0MemObjAddressR3(pChunk->paMappingsX[i].hMapObj);
4302 Log(("gmmR0MapChunk: chunk %#x is already mapped at %p!\n", pChunk->Core.Key, *ppvR3));
4303#ifdef VBOX_WITH_PAGE_SHARING
4304 /* The ring-3 chunk cache can be out of sync; don't fail. */
4305 return VINF_SUCCESS;
4306#else
4307 return VERR_GMM_CHUNK_ALREADY_MAPPED;
4308#endif
4309 }
4310 }
4311
4312 /*
4313 * Do the mapping.
4314 */
4315 RTR0MEMOBJ hMapObj;
4316 int rc = RTR0MemObjMapUser(&hMapObj, pChunk->hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
4317 if (RT_SUCCESS(rc))
4318 {
4319 /* reallocate the array? assumes few users per chunk (usually one). */
4320 unsigned iMapping = pChunk->cMappingsX;
4321 if ( iMapping <= 3
4322 || (iMapping & 3) == 0)
4323 {
4324 unsigned cNewSize = iMapping <= 3
4325 ? iMapping + 1
4326 : iMapping + 4;
4327 Assert(cNewSize < 4 || RT_ALIGN_32(cNewSize, 4) == cNewSize);
4328 if (RT_UNLIKELY(cNewSize > UINT16_MAX))
4329 {
4330 rc = RTR0MemObjFree(hMapObj, false /* fFreeMappings (NA) */); AssertRC(rc);
4331 return VERR_GMM_TOO_MANY_CHUNK_MAPPINGS;
4332 }
4333
4334 void *pvMappings = RTMemRealloc(pChunk->paMappingsX, cNewSize * sizeof(pChunk->paMappingsX[0]));
4335 if (RT_UNLIKELY(!pvMappings))
4336 {
4337 rc = RTR0MemObjFree(hMapObj, false /* fFreeMappings (NA) */); AssertRC(rc);
4338 return VERR_NO_MEMORY;
4339 }
4340 pChunk->paMappingsX = (PGMMCHUNKMAP)pvMappings;
4341 }
4342
4343 /* insert new entry */
4344 pChunk->paMappingsX[iMapping].hMapObj = hMapObj;
4345 pChunk->paMappingsX[iMapping].pGVM = pGVM;
4346 Assert(pChunk->cMappingsX == iMapping);
4347 pChunk->cMappingsX = iMapping + 1;
4348
4349 *ppvR3 = RTR0MemObjAddressR3(hMapObj);
4350 }
4351
4352 return rc;
4353}
4354
4355
4356/**
4357 * Maps a chunk into the user address space of the current process.
4358 *
4359 * @returns VBox status code.
4360 * @param pGMM Pointer to the GMM instance data.
4361 * @param pGVM Pointer to the Global VM structure.
4362 * @param pChunk Pointer to the chunk to be mapped.
4363 * @param fRelaxedSem Whether we can release the semaphore while doing the
4364 * mapping (@c true) or not.
4365 * @param ppvR3 Where to store the ring-3 address of the mapping.
4366 * In the VERR_GMM_CHUNK_ALREADY_MAPPED case, this will be
4367 * contain the address of the existing mapping.
4368 */
4369static int gmmR0MapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem, PRTR3PTR ppvR3)
4370{
4371 /*
4372 * Take the chunk lock and leave the giant GMM lock when possible, then
4373 * call the worker function.
4374 */
4375 GMMR0CHUNKMTXSTATE MtxState;
4376 int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk,
4377 fRelaxedSem ? GMMR0CHUNK_MTX_RETAKE_GIANT : GMMR0CHUNK_MTX_KEEP_GIANT);
4378 if (RT_SUCCESS(rc))
4379 {
4380 rc = gmmR0MapChunkLocked(pGMM, pGVM, pChunk, ppvR3);
4381 gmmR0ChunkMutexRelease(&MtxState, pChunk);
4382 }
4383
4384 return rc;
4385}
4386
4387
4388
4389#if defined(VBOX_WITH_PAGE_SHARING) || defined(VBOX_STRICT)
4390/**
4391 * Check if a chunk is mapped into the specified VM
4392 *
4393 * @returns mapped yes/no
4394 * @param pGMM Pointer to the GMM instance.
4395 * @param pGVM Pointer to the Global VM structure.
4396 * @param pChunk Pointer to the chunk to be mapped.
4397 * @param ppvR3 Where to store the ring-3 address of the mapping.
4398 */
4399static bool gmmR0IsChunkMapped(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3)
4400{
4401 GMMR0CHUNKMTXSTATE MtxState;
4402 gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT);
4403 for (uint32_t i = 0; i < pChunk->cMappingsX; i++)
4404 {
4405 Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ);
4406 if (pChunk->paMappingsX[i].pGVM == pGVM)
4407 {
4408 *ppvR3 = RTR0MemObjAddressR3(pChunk->paMappingsX[i].hMapObj);
4409 gmmR0ChunkMutexRelease(&MtxState, pChunk);
4410 return true;
4411 }
4412 }
4413 *ppvR3 = NULL;
4414 gmmR0ChunkMutexRelease(&MtxState, pChunk);
4415 return false;
4416}
4417#endif /* VBOX_WITH_PAGE_SHARING || VBOX_STRICT */
4418
4419
4420/**
4421 * Map a chunk and/or unmap another chunk.
4422 *
4423 * The mapping and unmapping applies to the current process.
4424 *
4425 * This API does two things because it saves a kernel call per mapping when
4426 * when the ring-3 mapping cache is full.
4427 *
4428 * @returns VBox status code.
4429 * @param pGVM The global (ring-0) VM structure.
4430 * @param idChunkMap The chunk to map. NIL_GMM_CHUNKID if nothing to map.
4431 * @param idChunkUnmap The chunk to unmap. NIL_GMM_CHUNKID if nothing to unmap.
4432 * @param ppvR3 Where to store the address of the mapped chunk. NULL is ok if nothing to map.
4433 * @thread EMT ???
4434 */
4435GMMR0DECL(int) GMMR0MapUnmapChunk(PGVM pGVM, uint32_t idChunkMap, uint32_t idChunkUnmap, PRTR3PTR ppvR3)
4436{
4437 LogFlow(("GMMR0MapUnmapChunk: pGVM=%p idChunkMap=%#x idChunkUnmap=%#x ppvR3=%p\n",
4438 pGVM, idChunkMap, idChunkUnmap, ppvR3));
4439
4440 /*
4441 * Validate input and get the basics.
4442 */
4443 PGMM pGMM;
4444 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4445 int rc = GVMMR0ValidateGVM(pGVM);
4446 if (RT_FAILURE(rc))
4447 return rc;
4448
4449 AssertCompile(NIL_GMM_CHUNKID == 0);
4450 AssertMsgReturn(idChunkMap <= GMM_CHUNKID_LAST, ("%#x\n", idChunkMap), VERR_INVALID_PARAMETER);
4451 AssertMsgReturn(idChunkUnmap <= GMM_CHUNKID_LAST, ("%#x\n", idChunkUnmap), VERR_INVALID_PARAMETER);
4452
4453 if ( idChunkMap == NIL_GMM_CHUNKID
4454 && idChunkUnmap == NIL_GMM_CHUNKID)
4455 return VERR_INVALID_PARAMETER;
4456
4457 if (idChunkMap != NIL_GMM_CHUNKID)
4458 {
4459 AssertPtrReturn(ppvR3, VERR_INVALID_POINTER);
4460 *ppvR3 = NIL_RTR3PTR;
4461 }
4462
4463 /*
4464 * Take the semaphore and do the work.
4465 *
4466 * The unmapping is done last since it's easier to undo a mapping than
4467 * undoing an unmapping. The ring-3 mapping cache cannot not be so big
4468 * that it pushes the user virtual address space to within a chunk of
4469 * it it's limits, so, no problem here.
4470 */
4471 gmmR0MutexAcquire(pGMM);
4472 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
4473 {
4474 PGMMCHUNK pMap = NULL;
4475 if (idChunkMap != NIL_GVM_HANDLE)
4476 {
4477 pMap = gmmR0GetChunk(pGMM, idChunkMap);
4478 if (RT_LIKELY(pMap))
4479 rc = gmmR0MapChunk(pGMM, pGVM, pMap, true /*fRelaxedSem*/, ppvR3);
4480 else
4481 {
4482 Log(("GMMR0MapUnmapChunk: idChunkMap=%#x\n", idChunkMap));
4483 rc = VERR_GMM_CHUNK_NOT_FOUND;
4484 }
4485 }
4486/** @todo split this operation, the bail out might (theoretcially) not be
4487 * entirely safe. */
4488
4489 if ( idChunkUnmap != NIL_GMM_CHUNKID
4490 && RT_SUCCESS(rc))
4491 {
4492 PGMMCHUNK pUnmap = gmmR0GetChunk(pGMM, idChunkUnmap);
4493 if (RT_LIKELY(pUnmap))
4494 rc = gmmR0UnmapChunk(pGMM, pGVM, pUnmap, true /*fRelaxedSem*/);
4495 else
4496 {
4497 Log(("GMMR0MapUnmapChunk: idChunkUnmap=%#x\n", idChunkUnmap));
4498 rc = VERR_GMM_CHUNK_NOT_FOUND;
4499 }
4500
4501 if (RT_FAILURE(rc) && pMap)
4502 gmmR0UnmapChunk(pGMM, pGVM, pMap, false /*fRelaxedSem*/);
4503 }
4504
4505 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
4506 }
4507 else
4508 rc = VERR_GMM_IS_NOT_SANE;
4509 gmmR0MutexRelease(pGMM);
4510
4511 LogFlow(("GMMR0MapUnmapChunk: returns %Rrc\n", rc));
4512 return rc;
4513}
4514
4515
4516/**
4517 * VMMR0 request wrapper for GMMR0MapUnmapChunk.
4518 *
4519 * @returns see GMMR0MapUnmapChunk.
4520 * @param pGVM The global (ring-0) VM structure.
4521 * @param pReq Pointer to the request packet.
4522 */
4523GMMR0DECL(int) GMMR0MapUnmapChunkReq(PGVM pGVM, PGMMMAPUNMAPCHUNKREQ pReq)
4524{
4525 /*
4526 * Validate input and pass it on.
4527 */
4528 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
4529 AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
4530
4531 return GMMR0MapUnmapChunk(pGVM, pReq->idChunkMap, pReq->idChunkUnmap, &pReq->pvR3);
4532}
4533
4534
4535#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
4536/**
4537 * Gets the ring-0 virtual address for the given page.
4538 *
4539 * This is used by PGM when IEM and such wants to access guest RAM from ring-0.
4540 * One of the ASSUMPTIONS here is that the @a idPage is used by the VM and the
4541 * corresponding chunk will remain valid beyond the call (at least till the EMT
4542 * returns to ring-3).
4543 *
4544 * @returns VBox status code.
4545 * @param pGVM Pointer to the kernel-only VM instace data.
4546 * @param idPage The page ID.
4547 * @param ppv Where to store the address.
4548 * @thread EMT
4549 */
4550GMMR0DECL(int) GMMR0PageIdToVirt(PGVM pGVM, uint32_t idPage, void **ppv)
4551{
4552 *ppv = NULL;
4553 PGMM pGMM;
4554 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4555
4556 uint32_t const idChunk = idPage >> GMM_CHUNKID_SHIFT;
4557
4558 /*
4559 * Start with the per-VM TLB.
4560 */
4561 RTSpinlockAcquire(pGVM->gmm.s.hChunkTlbSpinLock);
4562
4563 PGMMPERVMCHUNKTLBE pTlbe = &pGVM->gmm.s.aChunkTlbEntries[GMMPERVM_CHUNKTLB_IDX(idChunk)];
4564 PGMMCHUNK pChunk = pTlbe->pChunk;
4565 if ( pChunk != NULL
4566 && pTlbe->idGeneration == ASMAtomicUoReadU64(&pGMM->idFreeGeneration)
4567 && pChunk->Core.Key == idChunk)
4568 pGVM->R0Stats.gmm.cChunkTlbHits++; /* hopefully this is a likely outcome */
4569 else
4570 {
4571 pGVM->R0Stats.gmm.cChunkTlbMisses++;
4572
4573 /*
4574 * Look it up in the chunk tree.
4575 */
4576 RTSpinlockAcquire(pGMM->hSpinLockTree);
4577 pChunk = gmmR0GetChunkLocked(pGMM, idChunk);
4578 if (RT_LIKELY(pChunk))
4579 {
4580 pTlbe->idGeneration = pGMM->idFreeGeneration;
4581 RTSpinlockRelease(pGMM->hSpinLockTree);
4582 pTlbe->pChunk = pChunk;
4583 }
4584 else
4585 {
4586 RTSpinlockRelease(pGMM->hSpinLockTree);
4587 RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock);
4588 AssertMsgFailed(("idPage=%#x\n", idPage));
4589 return VERR_GMM_PAGE_NOT_FOUND;
4590 }
4591 }
4592
4593 RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock);
4594
4595 /*
4596 * Got a chunk, now validate the page ownership and calcuate it's address.
4597 */
4598 const GMMPAGE * const pPage = &pChunk->aPages[idPage & GMM_PAGEID_IDX_MASK];
4599 if (RT_LIKELY( ( GMM_PAGE_IS_PRIVATE(pPage)
4600 && pPage->Private.hGVM == pGVM->hSelf)
4601 || GMM_PAGE_IS_SHARED(pPage)))
4602 {
4603 AssertPtr(pChunk->pbMapping);
4604 *ppv = &pChunk->pbMapping[(idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT];
4605 return VINF_SUCCESS;
4606 }
4607 AssertMsgFailed(("idPage=%#x is-private=%RTbool Private.hGVM=%u pGVM->hGVM=%u\n",
4608 idPage, GMM_PAGE_IS_PRIVATE(pPage), pPage->Private.hGVM, pGVM->hSelf));
4609 return VERR_GMM_NOT_PAGE_OWNER;
4610}
4611#endif /* !VBOX_WITH_LINEAR_HOST_PHYS_MEM */
4612
4613#ifdef VBOX_WITH_PAGE_SHARING
4614
4615# ifdef VBOX_STRICT
4616/**
4617 * For checksumming shared pages in strict builds.
4618 *
4619 * The purpose is making sure that a page doesn't change.
4620 *
4621 * @returns Checksum, 0 on failure.
4622 * @param pGMM The GMM instance data.
4623 * @param pGVM Pointer to the kernel-only VM instace data.
4624 * @param idPage The page ID.
4625 */
4626static uint32_t gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage)
4627{
4628 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
4629 AssertMsgReturn(pChunk, ("idPage=%#x\n", idPage), 0);
4630
4631 uint8_t *pbChunk;
4632 if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
4633 return 0;
4634 uint8_t const *pbPage = pbChunk + ((idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
4635
4636 return RTCrc32(pbPage, GUEST_PAGE_SIZE);
4637}
4638# endif /* VBOX_STRICT */
4639
4640
4641/**
4642 * Calculates the module hash value.
4643 *
4644 * @returns Hash value.
4645 * @param pszModuleName The module name.
4646 * @param pszVersion The module version string.
4647 */
4648static uint32_t gmmR0ShModCalcHash(const char *pszModuleName, const char *pszVersion)
4649{
4650 return RTStrHash1ExN(3, pszModuleName, RTSTR_MAX, "::", (size_t)2, pszVersion, RTSTR_MAX);
4651}
4652
4653
4654/**
4655 * Finds a global module.
4656 *
4657 * @returns Pointer to the global module on success, NULL if not found.
4658 * @param pGMM The GMM instance data.
4659 * @param uHash The hash as calculated by gmmR0ShModCalcHash.
4660 * @param cbModule The module size.
4661 * @param enmGuestOS The guest OS type.
4662 * @param cRegions The number of regions.
4663 * @param pszModuleName The module name.
4664 * @param pszVersion The module version.
4665 * @param paRegions The region descriptions.
4666 */
4667static PGMMSHAREDMODULE gmmR0ShModFindGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS,
4668 uint32_t cRegions, const char *pszModuleName, const char *pszVersion,
4669 struct VMMDEVSHAREDREGIONDESC const *paRegions)
4670{
4671 for (PGMMSHAREDMODULE pGblMod = (PGMMSHAREDMODULE)RTAvllU32Get(&pGMM->pGlobalSharedModuleTree, uHash);
4672 pGblMod;
4673 pGblMod = (PGMMSHAREDMODULE)pGblMod->Core.pList)
4674 {
4675 if (pGblMod->cbModule != cbModule)
4676 continue;
4677 if (pGblMod->enmGuestOS != enmGuestOS)
4678 continue;
4679 if (pGblMod->cRegions != cRegions)
4680 continue;
4681 if (strcmp(pGblMod->szName, pszModuleName))
4682 continue;
4683 if (strcmp(pGblMod->szVersion, pszVersion))
4684 continue;
4685
4686 uint32_t i;
4687 for (i = 0; i < cRegions; i++)
4688 {
4689 uint32_t off = paRegions[i].GCRegionAddr & GUEST_PAGE_OFFSET_MASK;
4690 if (pGblMod->aRegions[i].off != off)
4691 break;
4692
4693 uint32_t cb = RT_ALIGN_32(paRegions[i].cbRegion + off, GUEST_PAGE_SIZE);
4694 if (pGblMod->aRegions[i].cb != cb)
4695 break;
4696 }
4697
4698 if (i == cRegions)
4699 return pGblMod;
4700 }
4701
4702 return NULL;
4703}
4704
4705
4706/**
4707 * Creates a new global module.
4708 *
4709 * @returns VBox status code.
4710 * @param pGMM The GMM instance data.
4711 * @param uHash The hash as calculated by gmmR0ShModCalcHash.
4712 * @param cbModule The module size.
4713 * @param enmGuestOS The guest OS type.
4714 * @param cRegions The number of regions.
4715 * @param pszModuleName The module name.
4716 * @param pszVersion The module version.
4717 * @param paRegions The region descriptions.
4718 * @param ppGblMod Where to return the new module on success.
4719 */
4720static int gmmR0ShModNewGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS,
4721 uint32_t cRegions, const char *pszModuleName, const char *pszVersion,
4722 struct VMMDEVSHAREDREGIONDESC const *paRegions, PGMMSHAREDMODULE *ppGblMod)
4723{
4724 Log(("gmmR0ShModNewGlobal: %s %s size %#x os %u rgn %u\n", pszModuleName, pszVersion, cbModule, enmGuestOS, cRegions));
4725 if (pGMM->cShareableModules >= GMM_MAX_SHARED_GLOBAL_MODULES)
4726 {
4727 Log(("gmmR0ShModNewGlobal: Too many modules\n"));
4728 return VERR_GMM_TOO_MANY_GLOBAL_MODULES;
4729 }
4730
4731 PGMMSHAREDMODULE pGblMod = (PGMMSHAREDMODULE)RTMemAllocZ(RT_UOFFSETOF_DYN(GMMSHAREDMODULE, aRegions[cRegions]));
4732 if (!pGblMod)
4733 {
4734 Log(("gmmR0ShModNewGlobal: No memory\n"));
4735 return VERR_NO_MEMORY;
4736 }
4737
4738 pGblMod->Core.Key = uHash;
4739 pGblMod->cbModule = cbModule;
4740 pGblMod->cRegions = cRegions;
4741 pGblMod->cUsers = 1;
4742 pGblMod->enmGuestOS = enmGuestOS;
4743 strcpy(pGblMod->szName, pszModuleName);
4744 strcpy(pGblMod->szVersion, pszVersion);
4745
4746 for (uint32_t i = 0; i < cRegions; i++)
4747 {
4748 Log(("gmmR0ShModNewGlobal: rgn[%u]=%RGvLB%#x\n", i, paRegions[i].GCRegionAddr, paRegions[i].cbRegion));
4749 pGblMod->aRegions[i].off = paRegions[i].GCRegionAddr & GUEST_PAGE_OFFSET_MASK;
4750 pGblMod->aRegions[i].cb = paRegions[i].cbRegion + pGblMod->aRegions[i].off;
4751 pGblMod->aRegions[i].cb = RT_ALIGN_32(pGblMod->aRegions[i].cb, GUEST_PAGE_SIZE);
4752 pGblMod->aRegions[i].paidPages = NULL; /* allocated when needed. */
4753 }
4754
4755 bool fInsert = RTAvllU32Insert(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core);
4756 Assert(fInsert); NOREF(fInsert);
4757 pGMM->cShareableModules++;
4758
4759 *ppGblMod = pGblMod;
4760 return VINF_SUCCESS;
4761}
4762
4763
4764/**
4765 * Deletes a global module which is no longer referenced by anyone.
4766 *
4767 * @param pGMM The GMM instance data.
4768 * @param pGblMod The module to delete.
4769 */
4770static void gmmR0ShModDeleteGlobal(PGMM pGMM, PGMMSHAREDMODULE pGblMod)
4771{
4772 Assert(pGblMod->cUsers == 0);
4773 Assert(pGMM->cShareableModules > 0 && pGMM->cShareableModules <= GMM_MAX_SHARED_GLOBAL_MODULES);
4774
4775 void *pvTest = RTAvllU32RemoveNode(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core);
4776 Assert(pvTest == pGblMod); NOREF(pvTest);
4777 pGMM->cShareableModules--;
4778
4779 uint32_t i = pGblMod->cRegions;
4780 while (i-- > 0)
4781 {
4782 if (pGblMod->aRegions[i].paidPages)
4783 {
4784 /* We don't doing anything to the pages as they are handled by the
4785 copy-on-write mechanism in PGM. */
4786 RTMemFree(pGblMod->aRegions[i].paidPages);
4787 pGblMod->aRegions[i].paidPages = NULL;
4788 }
4789 }
4790 RTMemFree(pGblMod);
4791}
4792
4793
4794static int gmmR0ShModNewPerVM(PGVM pGVM, RTGCPTR GCBaseAddr, uint32_t cRegions, const VMMDEVSHAREDREGIONDESC *paRegions,
4795 PGMMSHAREDMODULEPERVM *ppRecVM)
4796{
4797 if (pGVM->gmm.s.Stats.cShareableModules >= GMM_MAX_SHARED_PER_VM_MODULES)
4798 return VERR_GMM_TOO_MANY_PER_VM_MODULES;
4799
4800 PGMMSHAREDMODULEPERVM pRecVM;
4801 pRecVM = (PGMMSHAREDMODULEPERVM)RTMemAllocZ(RT_UOFFSETOF_DYN(GMMSHAREDMODULEPERVM, aRegionsGCPtrs[cRegions]));
4802 if (!pRecVM)
4803 return VERR_NO_MEMORY;
4804
4805 pRecVM->Core.Key = GCBaseAddr;
4806 for (uint32_t i = 0; i < cRegions; i++)
4807 pRecVM->aRegionsGCPtrs[i] = paRegions[i].GCRegionAddr;
4808
4809 bool fInsert = RTAvlGCPtrInsert(&pGVM->gmm.s.pSharedModuleTree, &pRecVM->Core);
4810 Assert(fInsert); NOREF(fInsert);
4811 pGVM->gmm.s.Stats.cShareableModules++;
4812
4813 *ppRecVM = pRecVM;
4814 return VINF_SUCCESS;
4815}
4816
4817
4818static void gmmR0ShModDeletePerVM(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULEPERVM pRecVM, bool fRemove)
4819{
4820 /*
4821 * Free the per-VM module.
4822 */
4823 PGMMSHAREDMODULE pGblMod = pRecVM->pGlobalModule;
4824 pRecVM->pGlobalModule = NULL;
4825
4826 if (fRemove)
4827 {
4828 void *pvTest = RTAvlGCPtrRemove(&pGVM->gmm.s.pSharedModuleTree, pRecVM->Core.Key);
4829 Assert(pvTest == &pRecVM->Core); NOREF(pvTest);
4830 }
4831
4832 RTMemFree(pRecVM);
4833
4834 /*
4835 * Release the global module.
4836 * (In the registration bailout case, it might not be.)
4837 */
4838 if (pGblMod)
4839 {
4840 Assert(pGblMod->cUsers > 0);
4841 pGblMod->cUsers--;
4842 if (pGblMod->cUsers == 0)
4843 gmmR0ShModDeleteGlobal(pGMM, pGblMod);
4844 }
4845}
4846
4847#endif /* VBOX_WITH_PAGE_SHARING */
4848
4849/**
4850 * Registers a new shared module for the VM.
4851 *
4852 * @returns VBox status code.
4853 * @param pGVM The global (ring-0) VM structure.
4854 * @param idCpu The VCPU id.
4855 * @param enmGuestOS The guest OS type.
4856 * @param pszModuleName The module name.
4857 * @param pszVersion The module version.
4858 * @param GCPtrModBase The module base address.
4859 * @param cbModule The module size.
4860 * @param cRegions The mumber of shared region descriptors.
4861 * @param paRegions Pointer to an array of shared region(s).
4862 * @thread EMT(idCpu)
4863 */
4864GMMR0DECL(int) GMMR0RegisterSharedModule(PGVM pGVM, VMCPUID idCpu, VBOXOSFAMILY enmGuestOS, char *pszModuleName,
4865 char *pszVersion, RTGCPTR GCPtrModBase, uint32_t cbModule,
4866 uint32_t cRegions, struct VMMDEVSHAREDREGIONDESC const *paRegions)
4867{
4868#ifdef VBOX_WITH_PAGE_SHARING
4869 /*
4870 * Validate input and get the basics.
4871 *
4872 * Note! Turns out the module size does necessarily match the size of the
4873 * regions. (iTunes on XP)
4874 */
4875 PGMM pGMM;
4876 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
4877 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
4878 if (RT_FAILURE(rc))
4879 return rc;
4880
4881 if (RT_UNLIKELY(cRegions > VMMDEVSHAREDREGIONDESC_MAX))
4882 return VERR_GMM_TOO_MANY_REGIONS;
4883
4884 if (RT_UNLIKELY(cbModule == 0 || cbModule > _1G))
4885 return VERR_GMM_BAD_SHARED_MODULE_SIZE;
4886
4887 uint32_t cbTotal = 0;
4888 for (uint32_t i = 0; i < cRegions; i++)
4889 {
4890 if (RT_UNLIKELY(paRegions[i].cbRegion == 0 || paRegions[i].cbRegion > _1G))
4891 return VERR_GMM_SHARED_MODULE_BAD_REGIONS_SIZE;
4892
4893 cbTotal += paRegions[i].cbRegion;
4894 if (RT_UNLIKELY(cbTotal > _1G))
4895 return VERR_GMM_SHARED_MODULE_BAD_REGIONS_SIZE;
4896 }
4897
4898 AssertPtrReturn(pszModuleName, VERR_INVALID_POINTER);
4899 if (RT_UNLIKELY(!memchr(pszModuleName, '\0', GMM_SHARED_MODULE_MAX_NAME_STRING)))
4900 return VERR_GMM_MODULE_NAME_TOO_LONG;
4901
4902 AssertPtrReturn(pszVersion, VERR_INVALID_POINTER);
4903 if (RT_UNLIKELY(!memchr(pszVersion, '\0', GMM_SHARED_MODULE_MAX_VERSION_STRING)))
4904 return VERR_GMM_MODULE_NAME_TOO_LONG;
4905
4906 uint32_t const uHash = gmmR0ShModCalcHash(pszModuleName, pszVersion);
4907 Log(("GMMR0RegisterSharedModule %s %s base %RGv size %x hash %x\n", pszModuleName, pszVersion, GCPtrModBase, cbModule, uHash));
4908
4909 /*
4910 * Take the semaphore and do some more validations.
4911 */
4912 gmmR0MutexAcquire(pGMM);
4913 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
4914 {
4915 /*
4916 * Check if this module is already locally registered and register
4917 * it if it isn't. The base address is a unique module identifier
4918 * locally.
4919 */
4920 PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)RTAvlGCPtrGet(&pGVM->gmm.s.pSharedModuleTree, GCPtrModBase);
4921 bool fNewModule = pRecVM == NULL;
4922 if (fNewModule)
4923 {
4924 rc = gmmR0ShModNewPerVM(pGVM, GCPtrModBase, cRegions, paRegions, &pRecVM);
4925 if (RT_SUCCESS(rc))
4926 {
4927 /*
4928 * Find a matching global module, register a new one if needed.
4929 */
4930 PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
4931 pszModuleName, pszVersion, paRegions);
4932 if (!pGblMod)
4933 {
4934 Assert(fNewModule);
4935 rc = gmmR0ShModNewGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
4936 pszModuleName, pszVersion, paRegions, &pGblMod);
4937 if (RT_SUCCESS(rc))
4938 {
4939 pRecVM->pGlobalModule = pGblMod; /* (One referenced returned by gmmR0ShModNewGlobal.) */
4940 Log(("GMMR0RegisterSharedModule: new module %s %s\n", pszModuleName, pszVersion));
4941 }
4942 else
4943 gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/);
4944 }
4945 else
4946 {
4947 Assert(pGblMod->cUsers > 0 && pGblMod->cUsers < UINT32_MAX / 2);
4948 pGblMod->cUsers++;
4949 pRecVM->pGlobalModule = pGblMod;
4950
4951 Log(("GMMR0RegisterSharedModule: new per vm module %s %s, gbl users %d\n", pszModuleName, pszVersion, pGblMod->cUsers));
4952 }
4953 }
4954 }
4955 else
4956 {
4957 /*
4958 * Attempt to re-register an existing module.
4959 */
4960 PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions,
4961 pszModuleName, pszVersion, paRegions);
4962 if (pRecVM->pGlobalModule == pGblMod)
4963 {
4964 Log(("GMMR0RegisterSharedModule: already registered %s %s, gbl users %d\n", pszModuleName, pszVersion, pGblMod->cUsers));
4965 rc = VINF_GMM_SHARED_MODULE_ALREADY_REGISTERED;
4966 }
4967 else
4968 {
4969 /** @todo may have to unregister+register when this happens in case it's caused
4970 * by VBoxService crashing and being restarted... */
4971 Log(("GMMR0RegisterSharedModule: Address clash!\n"
4972 " incoming at %RGvLB%#x %s %s rgns %u\n"
4973 " existing at %RGvLB%#x %s %s rgns %u\n",
4974 GCPtrModBase, cbModule, pszModuleName, pszVersion, cRegions,
4975 pRecVM->Core.Key, pRecVM->pGlobalModule->cbModule, pRecVM->pGlobalModule->szName,
4976 pRecVM->pGlobalModule->szVersion, pRecVM->pGlobalModule->cRegions));
4977 rc = VERR_GMM_SHARED_MODULE_ADDRESS_CLASH;
4978 }
4979 }
4980 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
4981 }
4982 else
4983 rc = VERR_GMM_IS_NOT_SANE;
4984
4985 gmmR0MutexRelease(pGMM);
4986 return rc;
4987#else
4988
4989 NOREF(pGVM); NOREF(idCpu); NOREF(enmGuestOS); NOREF(pszModuleName); NOREF(pszVersion);
4990 NOREF(GCPtrModBase); NOREF(cbModule); NOREF(cRegions); NOREF(paRegions);
4991 return VERR_NOT_IMPLEMENTED;
4992#endif
4993}
4994
4995
4996/**
4997 * VMMR0 request wrapper for GMMR0RegisterSharedModule.
4998 *
4999 * @returns see GMMR0RegisterSharedModule.
5000 * @param pGVM The global (ring-0) VM structure.
5001 * @param idCpu The VCPU id.
5002 * @param pReq Pointer to the request packet.
5003 */
5004GMMR0DECL(int) GMMR0RegisterSharedModuleReq(PGVM pGVM, VMCPUID idCpu, PGMMREGISTERSHAREDMODULEREQ pReq)
5005{
5006 /*
5007 * Validate input and pass it on.
5008 */
5009 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
5010 AssertMsgReturn( pReq->Hdr.cbReq >= sizeof(*pReq)
5011 && pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMREGISTERSHAREDMODULEREQ, aRegions[pReq->cRegions]),
5012 ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
5013
5014 /* Pass back return code in the request packet to preserve informational codes. (VMMR3CallR0 chokes on them) */
5015 pReq->rc = GMMR0RegisterSharedModule(pGVM, idCpu, pReq->enmGuestOS, pReq->szName, pReq->szVersion,
5016 pReq->GCBaseAddr, pReq->cbModule, pReq->cRegions, pReq->aRegions);
5017 return VINF_SUCCESS;
5018}
5019
5020
5021/**
5022 * Unregisters a shared module for the VM
5023 *
5024 * @returns VBox status code.
5025 * @param pGVM The global (ring-0) VM structure.
5026 * @param idCpu The VCPU id.
5027 * @param pszModuleName The module name.
5028 * @param pszVersion The module version.
5029 * @param GCPtrModBase The module base address.
5030 * @param cbModule The module size.
5031 */
5032GMMR0DECL(int) GMMR0UnregisterSharedModule(PGVM pGVM, VMCPUID idCpu, char *pszModuleName, char *pszVersion,
5033 RTGCPTR GCPtrModBase, uint32_t cbModule)
5034{
5035#ifdef VBOX_WITH_PAGE_SHARING
5036 /*
5037 * Validate input and get the basics.
5038 */
5039 PGMM pGMM;
5040 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5041 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
5042 if (RT_FAILURE(rc))
5043 return rc;
5044
5045 AssertPtrReturn(pszModuleName, VERR_INVALID_POINTER);
5046 AssertPtrReturn(pszVersion, VERR_INVALID_POINTER);
5047 if (RT_UNLIKELY(!memchr(pszModuleName, '\0', GMM_SHARED_MODULE_MAX_NAME_STRING)))
5048 return VERR_GMM_MODULE_NAME_TOO_LONG;
5049 if (RT_UNLIKELY(!memchr(pszVersion, '\0', GMM_SHARED_MODULE_MAX_VERSION_STRING)))
5050 return VERR_GMM_MODULE_NAME_TOO_LONG;
5051
5052 Log(("GMMR0UnregisterSharedModule %s %s base=%RGv size %x\n", pszModuleName, pszVersion, GCPtrModBase, cbModule));
5053
5054 /*
5055 * Take the semaphore and do some more validations.
5056 */
5057 gmmR0MutexAcquire(pGMM);
5058 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
5059 {
5060 /*
5061 * Locate and remove the specified module.
5062 */
5063 PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)RTAvlGCPtrGet(&pGVM->gmm.s.pSharedModuleTree, GCPtrModBase);
5064 if (pRecVM)
5065 {
5066 /** @todo Do we need to do more validations here, like that the
5067 * name + version + cbModule matches? */
5068 NOREF(cbModule);
5069 Assert(pRecVM->pGlobalModule);
5070 gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/);
5071 }
5072 else
5073 rc = VERR_GMM_SHARED_MODULE_NOT_FOUND;
5074
5075 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
5076 }
5077 else
5078 rc = VERR_GMM_IS_NOT_SANE;
5079
5080 gmmR0MutexRelease(pGMM);
5081 return rc;
5082#else
5083
5084 NOREF(pGVM); NOREF(idCpu); NOREF(pszModuleName); NOREF(pszVersion); NOREF(GCPtrModBase); NOREF(cbModule);
5085 return VERR_NOT_IMPLEMENTED;
5086#endif
5087}
5088
5089
5090/**
5091 * VMMR0 request wrapper for GMMR0UnregisterSharedModule.
5092 *
5093 * @returns see GMMR0UnregisterSharedModule.
5094 * @param pGVM The global (ring-0) VM structure.
5095 * @param idCpu The VCPU id.
5096 * @param pReq Pointer to the request packet.
5097 */
5098GMMR0DECL(int) GMMR0UnregisterSharedModuleReq(PGVM pGVM, VMCPUID idCpu, PGMMUNREGISTERSHAREDMODULEREQ pReq)
5099{
5100 /*
5101 * Validate input and pass it on.
5102 */
5103 AssertPtrReturn(pReq, VERR_INVALID_POINTER);
5104 AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
5105
5106 return GMMR0UnregisterSharedModule(pGVM, idCpu, pReq->szName, pReq->szVersion, pReq->GCBaseAddr, pReq->cbModule);
5107}
5108
5109#ifdef VBOX_WITH_PAGE_SHARING
5110
5111/**
5112 * Increase the use count of a shared page, the page is known to exist and be valid and such.
5113 *
5114 * @param pGMM Pointer to the GMM instance.
5115 * @param pGVM Pointer to the GVM instance.
5116 * @param pPage The page structure.
5117 */
5118DECLINLINE(void) gmmR0UseSharedPage(PGMM pGMM, PGVM pGVM, PGMMPAGE pPage)
5119{
5120 Assert(pGMM->cSharedPages > 0);
5121 Assert(pGMM->cAllocatedPages > 0);
5122
5123 pGMM->cDuplicatePages++;
5124
5125 pPage->Shared.cRefs++;
5126 pGVM->gmm.s.Stats.cSharedPages++;
5127 pGVM->gmm.s.Stats.Allocated.cBasePages++;
5128}
5129
5130
5131/**
5132 * Converts a private page to a shared page, the page is known to exist and be valid and such.
5133 *
5134 * @param pGMM Pointer to the GMM instance.
5135 * @param pGVM Pointer to the GVM instance.
5136 * @param HCPhys Host physical address
5137 * @param idPage The Page ID
5138 * @param pPage The page structure.
5139 * @param pPageDesc Shared page descriptor
5140 */
5141DECLINLINE(void) gmmR0ConvertToSharedPage(PGMM pGMM, PGVM pGVM, RTHCPHYS HCPhys, uint32_t idPage, PGMMPAGE pPage,
5142 PGMMSHAREDPAGEDESC pPageDesc)
5143{
5144 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT);
5145 Assert(pChunk);
5146 Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES);
5147 Assert(GMM_PAGE_IS_PRIVATE(pPage));
5148
5149 pChunk->cPrivate--;
5150 pChunk->cShared++;
5151
5152 pGMM->cSharedPages++;
5153
5154 pGVM->gmm.s.Stats.cSharedPages++;
5155 pGVM->gmm.s.Stats.cPrivatePages--;
5156
5157 /* Modify the page structure. */
5158 pPage->Shared.pfn = (uint32_t)(uint64_t)(HCPhys >> GUEST_PAGE_SHIFT);
5159 pPage->Shared.cRefs = 1;
5160#ifdef VBOX_STRICT
5161 pPageDesc->u32StrictChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage);
5162 pPage->Shared.u14Checksum = pPageDesc->u32StrictChecksum;
5163#else
5164 NOREF(pPageDesc);
5165 pPage->Shared.u14Checksum = 0;
5166#endif
5167 pPage->Shared.u2State = GMM_PAGE_STATE_SHARED;
5168}
5169
5170
5171static int gmmR0SharedModuleCheckPageFirstTime(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULE pModule,
5172 unsigned idxRegion, unsigned idxPage,
5173 PGMMSHAREDPAGEDESC pPageDesc, PGMMSHAREDREGIONDESC pGlobalRegion)
5174{
5175 NOREF(pModule);
5176
5177 /* Easy case: just change the internal page type. */
5178 PGMMPAGE pPage = gmmR0GetPage(pGMM, pPageDesc->idPage);
5179 AssertMsgReturn(pPage, ("idPage=%#x (GCPhys=%RGp HCPhys=%RHp idxRegion=%#x idxPage=%#x) #1\n",
5180 pPageDesc->idPage, pPageDesc->GCPhys, pPageDesc->HCPhys, idxRegion, idxPage),
5181 VERR_PGM_PHYS_INVALID_PAGE_ID);
5182 NOREF(idxRegion);
5183
5184 AssertMsg(pPageDesc->GCPhys == (pPage->Private.pfn << 12), ("desc %RGp gmm %RGp\n", pPageDesc->HCPhys, (pPage->Private.pfn << 12)));
5185
5186 gmmR0ConvertToSharedPage(pGMM, pGVM, pPageDesc->HCPhys, pPageDesc->idPage, pPage, pPageDesc);
5187
5188 /* Keep track of these references. */
5189 pGlobalRegion->paidPages[idxPage] = pPageDesc->idPage;
5190
5191 return VINF_SUCCESS;
5192}
5193
5194/**
5195 * Checks specified shared module range for changes
5196 *
5197 * Performs the following tasks:
5198 * - If a shared page is new, then it changes the GMM page type to shared and
5199 * returns it in the pPageDesc descriptor.
5200 * - If a shared page already exists, then it checks if the VM page is
5201 * identical and if so frees the VM page and returns the shared page in
5202 * pPageDesc descriptor.
5203 *
5204 * @remarks ASSUMES the caller has acquired the GMM semaphore!!
5205 *
5206 * @returns VBox status code.
5207 * @param pGVM Pointer to the GVM instance data.
5208 * @param pModule Module description
5209 * @param idxRegion Region index
5210 * @param idxPage Page index
5211 * @param pPageDesc Page descriptor
5212 */
5213GMMR0DECL(int) GMMR0SharedModuleCheckPage(PGVM pGVM, PGMMSHAREDMODULE pModule, uint32_t idxRegion, uint32_t idxPage,
5214 PGMMSHAREDPAGEDESC pPageDesc)
5215{
5216 int rc;
5217 PGMM pGMM;
5218 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5219 pPageDesc->u32StrictChecksum = 0;
5220
5221 AssertMsgReturn(idxRegion < pModule->cRegions,
5222 ("idxRegion=%#x cRegions=%#x %s %s\n", idxRegion, pModule->cRegions, pModule->szName, pModule->szVersion),
5223 VERR_INVALID_PARAMETER);
5224
5225 uint32_t const cPages = pModule->aRegions[idxRegion].cb >> GUEST_PAGE_SHIFT;
5226 AssertMsgReturn(idxPage < cPages,
5227 ("idxRegion=%#x cRegions=%#x %s %s\n", idxRegion, pModule->cRegions, pModule->szName, pModule->szVersion),
5228 VERR_INVALID_PARAMETER);
5229
5230 LogFlow(("GMMR0SharedModuleCheckRange %s base %RGv region %d idxPage %d\n", pModule->szName, pModule->Core.Key, idxRegion, idxPage));
5231
5232 /*
5233 * First time; create a page descriptor array.
5234 */
5235 PGMMSHAREDREGIONDESC pGlobalRegion = &pModule->aRegions[idxRegion];
5236 if (!pGlobalRegion->paidPages)
5237 {
5238 Log(("Allocate page descriptor array for %d pages\n", cPages));
5239 pGlobalRegion->paidPages = (uint32_t *)RTMemAlloc(cPages * sizeof(pGlobalRegion->paidPages[0]));
5240 AssertReturn(pGlobalRegion->paidPages, VERR_NO_MEMORY);
5241
5242 /* Invalidate all descriptors. */
5243 uint32_t i = cPages;
5244 while (i-- > 0)
5245 pGlobalRegion->paidPages[i] = NIL_GMM_PAGEID;
5246 }
5247
5248 /*
5249 * We've seen this shared page for the first time?
5250 */
5251 if (pGlobalRegion->paidPages[idxPage] == NIL_GMM_PAGEID)
5252 {
5253 Log(("New shared page guest %RGp host %RHp\n", pPageDesc->GCPhys, pPageDesc->HCPhys));
5254 return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion);
5255 }
5256
5257 /*
5258 * We've seen it before...
5259 */
5260 Log(("Replace existing page guest %RGp host %RHp id %#x -> id %#x\n",
5261 pPageDesc->GCPhys, pPageDesc->HCPhys, pPageDesc->idPage, pGlobalRegion->paidPages[idxPage]));
5262 Assert(pPageDesc->idPage != pGlobalRegion->paidPages[idxPage]);
5263
5264 /*
5265 * Get the shared page source.
5266 */
5267 PGMMPAGE pPage = gmmR0GetPage(pGMM, pGlobalRegion->paidPages[idxPage]);
5268 AssertMsgReturn(pPage, ("idPage=%#x (idxRegion=%#x idxPage=%#x) #2\n", pPageDesc->idPage, idxRegion, idxPage),
5269 VERR_PGM_PHYS_INVALID_PAGE_ID);
5270
5271 if (pPage->Common.u2State != GMM_PAGE_STATE_SHARED)
5272 {
5273 /*
5274 * Page was freed at some point; invalidate this entry.
5275 */
5276 /** @todo this isn't really bullet proof. */
5277 Log(("Old shared page was freed -> create a new one\n"));
5278 pGlobalRegion->paidPages[idxPage] = NIL_GMM_PAGEID;
5279 return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion);
5280 }
5281
5282 Log(("Replace existing page guest host %RHp -> %RHp\n", pPageDesc->HCPhys, ((uint64_t)pPage->Shared.pfn) << GUEST_PAGE_SHIFT));
5283
5284 /*
5285 * Calculate the virtual address of the local page.
5286 */
5287 PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pPageDesc->idPage >> GMM_CHUNKID_SHIFT);
5288 AssertMsgReturn(pChunk, ("idPage=%#x (idxRegion=%#x idxPage=%#x) #4\n", pPageDesc->idPage, idxRegion, idxPage),
5289 VERR_PGM_PHYS_INVALID_PAGE_ID);
5290
5291 uint8_t *pbChunk;
5292 AssertMsgReturn(gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk),
5293 ("idPage=%#x (idxRegion=%#x idxPage=%#x) #3\n", pPageDesc->idPage, idxRegion, idxPage),
5294 VERR_PGM_PHYS_INVALID_PAGE_ID);
5295 uint8_t *pbLocalPage = pbChunk + ((pPageDesc->idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
5296
5297 /*
5298 * Calculate the virtual address of the shared page.
5299 */
5300 pChunk = gmmR0GetChunk(pGMM, pGlobalRegion->paidPages[idxPage] >> GMM_CHUNKID_SHIFT);
5301 Assert(pChunk); /* can't fail as gmmR0GetPage succeeded. */
5302
5303 /*
5304 * Get the virtual address of the physical page; map the chunk into the VM
5305 * process if not already done.
5306 */
5307 if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
5308 {
5309 Log(("Map chunk into process!\n"));
5310 rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk);
5311 AssertRCReturn(rc, rc);
5312 }
5313 uint8_t *pbSharedPage = pbChunk + ((pGlobalRegion->paidPages[idxPage] & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
5314
5315#ifdef VBOX_STRICT
5316 pPageDesc->u32StrictChecksum = RTCrc32(pbSharedPage, GUEST_PAGE_SIZE);
5317 uint32_t uChecksum = pPageDesc->u32StrictChecksum & UINT32_C(0x00003fff);
5318 AssertMsg(!uChecksum || uChecksum == pPage->Shared.u14Checksum || !pPage->Shared.u14Checksum,
5319 ("%#x vs %#x - idPage=%#x - %s %s\n", uChecksum, pPage->Shared.u14Checksum,
5320 pGlobalRegion->paidPages[idxPage], pModule->szName, pModule->szVersion));
5321#endif
5322
5323 if (memcmp(pbSharedPage, pbLocalPage, GUEST_PAGE_SIZE))
5324 {
5325 Log(("Unexpected differences found between local and shared page; skip\n"));
5326 /* Signal to the caller that this one hasn't changed. */
5327 pPageDesc->idPage = NIL_GMM_PAGEID;
5328 return VINF_SUCCESS;
5329 }
5330
5331 /*
5332 * Free the old local page.
5333 */
5334 GMMFREEPAGEDESC PageDesc;
5335 PageDesc.idPage = pPageDesc->idPage;
5336 rc = gmmR0FreePages(pGMM, pGVM, 1, &PageDesc, GMMACCOUNT_BASE);
5337 AssertRCReturn(rc, rc);
5338
5339 gmmR0UseSharedPage(pGMM, pGVM, pPage);
5340
5341 /*
5342 * Pass along the new physical address & page id.
5343 */
5344 pPageDesc->HCPhys = ((uint64_t)pPage->Shared.pfn) << GUEST_PAGE_SHIFT;
5345 pPageDesc->idPage = pGlobalRegion->paidPages[idxPage];
5346
5347 return VINF_SUCCESS;
5348}
5349
5350
5351/**
5352 * RTAvlGCPtrDestroy callback.
5353 *
5354 * @returns 0 or VERR_GMM_INSTANCE.
5355 * @param pNode The node to destroy.
5356 * @param pvArgs Pointer to an argument packet.
5357 */
5358static DECLCALLBACK(int) gmmR0CleanupSharedModule(PAVLGCPTRNODECORE pNode, void *pvArgs)
5359{
5360 gmmR0ShModDeletePerVM(((GMMR0SHMODPERVMDTORARGS *)pvArgs)->pGMM,
5361 ((GMMR0SHMODPERVMDTORARGS *)pvArgs)->pGVM,
5362 (PGMMSHAREDMODULEPERVM)pNode,
5363 false /*fRemove*/);
5364 return VINF_SUCCESS;
5365}
5366
5367
5368/**
5369 * Used by GMMR0CleanupVM to clean up shared modules.
5370 *
5371 * This is called without taking the GMM lock so that it can be yielded as
5372 * needed here.
5373 *
5374 * @param pGMM The GMM handle.
5375 * @param pGVM The global VM handle.
5376 */
5377static void gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM)
5378{
5379 gmmR0MutexAcquire(pGMM);
5380 GMM_CHECK_SANITY_UPON_ENTERING(pGMM);
5381
5382 GMMR0SHMODPERVMDTORARGS Args;
5383 Args.pGVM = pGVM;
5384 Args.pGMM = pGMM;
5385 RTAvlGCPtrDestroy(&pGVM->gmm.s.pSharedModuleTree, gmmR0CleanupSharedModule, &Args);
5386
5387 AssertMsg(pGVM->gmm.s.Stats.cShareableModules == 0, ("%d\n", pGVM->gmm.s.Stats.cShareableModules));
5388 pGVM->gmm.s.Stats.cShareableModules = 0;
5389
5390 gmmR0MutexRelease(pGMM);
5391}
5392
5393#endif /* VBOX_WITH_PAGE_SHARING */
5394
5395/**
5396 * Removes all shared modules for the specified VM
5397 *
5398 * @returns VBox status code.
5399 * @param pGVM The global (ring-0) VM structure.
5400 * @param idCpu The VCPU id.
5401 */
5402GMMR0DECL(int) GMMR0ResetSharedModules(PGVM pGVM, VMCPUID idCpu)
5403{
5404#ifdef VBOX_WITH_PAGE_SHARING
5405 /*
5406 * Validate input and get the basics.
5407 */
5408 PGMM pGMM;
5409 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5410 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
5411 if (RT_FAILURE(rc))
5412 return rc;
5413
5414 /*
5415 * Take the semaphore and do some more validations.
5416 */
5417 gmmR0MutexAcquire(pGMM);
5418 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
5419 {
5420 Log(("GMMR0ResetSharedModules\n"));
5421 GMMR0SHMODPERVMDTORARGS Args;
5422 Args.pGVM = pGVM;
5423 Args.pGMM = pGMM;
5424 RTAvlGCPtrDestroy(&pGVM->gmm.s.pSharedModuleTree, gmmR0CleanupSharedModule, &Args);
5425 pGVM->gmm.s.Stats.cShareableModules = 0;
5426
5427 rc = VINF_SUCCESS;
5428 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
5429 }
5430 else
5431 rc = VERR_GMM_IS_NOT_SANE;
5432
5433 gmmR0MutexRelease(pGMM);
5434 return rc;
5435#else
5436 RT_NOREF(pGVM, idCpu);
5437 return VERR_NOT_IMPLEMENTED;
5438#endif
5439}
5440
5441#ifdef VBOX_WITH_PAGE_SHARING
5442
5443/**
5444 * Tree enumeration callback for checking a shared module.
5445 */
5446static DECLCALLBACK(int) gmmR0CheckSharedModule(PAVLGCPTRNODECORE pNode, void *pvUser)
5447{
5448 GMMCHECKSHAREDMODULEINFO *pArgs = (GMMCHECKSHAREDMODULEINFO*)pvUser;
5449 PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)pNode;
5450 PGMMSHAREDMODULE pGblMod = pRecVM->pGlobalModule;
5451
5452 Log(("gmmR0CheckSharedModule: check %s %s base=%RGv size=%x\n",
5453 pGblMod->szName, pGblMod->szVersion, pGblMod->Core.Key, pGblMod->cbModule));
5454
5455 int rc = PGMR0SharedModuleCheck(pArgs->pGVM, pArgs->pGVM, pArgs->idCpu, pGblMod, pRecVM->aRegionsGCPtrs);
5456 if (RT_FAILURE(rc))
5457 return rc;
5458 return VINF_SUCCESS;
5459}
5460
5461#endif /* VBOX_WITH_PAGE_SHARING */
5462
5463/**
5464 * Check all shared modules for the specified VM.
5465 *
5466 * @returns VBox status code.
5467 * @param pGVM The global (ring-0) VM structure.
5468 * @param idCpu The calling EMT number.
5469 * @thread EMT(idCpu)
5470 */
5471GMMR0DECL(int) GMMR0CheckSharedModules(PGVM pGVM, VMCPUID idCpu)
5472{
5473#ifdef VBOX_WITH_PAGE_SHARING
5474 /*
5475 * Validate input and get the basics.
5476 */
5477 PGMM pGMM;
5478 GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE);
5479 int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
5480 if (RT_FAILURE(rc))
5481 return rc;
5482
5483# ifndef DEBUG_sandervl
5484 /*
5485 * Take the semaphore and do some more validations.
5486 */
5487 gmmR0MutexAcquire(pGMM);
5488# endif
5489 if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM))
5490 {
5491 /*
5492 * Walk the tree, checking each module.
5493 */
5494 Log(("GMMR0CheckSharedModules\n"));
5495
5496 GMMCHECKSHAREDMODULEINFO Args;
5497 Args.pGVM = pGVM;
5498 Args.idCpu = idCpu;
5499 rc = RTAvlGCPtrDoWithAll(&pGVM->gmm.s.pSharedModuleTree, true /* fFromLeft */, gmmR0CheckSharedModule, &Args);
5500
5501 Log(("GMMR0CheckSharedModules done (rc=%Rrc)!\n", rc));
5502 GMM_CHECK_SANITY_UPON_LEAVING(pGMM);
5503 }
5504 else
5505 rc = VERR_GMM_IS_NOT_SANE;
5506
5507# ifndef DEBUG_sandervl
5508 gmmR0MutexRelease(pGMM);
5509# endif
5510 return rc;
5511#else
5512 RT_NOREF(pGVM, idCpu);
5513 return VERR_NOT_IMPLEMENTED;
5514#endif
5515}
5516
5517#ifdef VBOX_STRICT
5518
5519/**
5520 * Worker for GMMR0FindDuplicatePageReq.
5521 *
5522 * @returns true if duplicate, false if not.
5523 */
5524static bool gmmR0FindDupPageInChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, uint8_t const *pbSourcePage)
5525{
5526 bool fFoundDuplicate = false;
5527 /* Only take chunks not mapped into this VM process; not entirely correct. */
5528 uint8_t *pbChunk;
5529 if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk))
5530 {
5531 int rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk);
5532 if (RT_SUCCESS(rc))
5533 {
5534 /*
5535 * Look for duplicate pages
5536 */
5537 uintptr_t iPage = GMM_CHUNK_NUM_PAGES;
5538 while (iPage-- > 0)
5539 {
5540 if (GMM_PAGE_IS_PRIVATE(&pChunk->aPages[iPage]))
5541 {
5542 uint8_t *pbDestPage = pbChunk + (iPage << GUEST_PAGE_SHIFT);
5543 if (!memcmp(pbSourcePage, pbDestPage, GUEST_PAGE_SIZE))
5544 {
5545 fFoundDuplicate = true;
5546 break;
5547 }
5548 }
5549 }
5550 gmmR0UnmapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/);
5551 }
5552 }
5553 return fFoundDuplicate;
5554}
5555
5556
5557/**
5558 * Find a duplicate of the specified page in other active VMs
5559 *
5560 * @returns VBox status code.
5561 * @param pGVM The global (ring-0) VM structure.
5562 * @param pReq Pointer to the request packet.
5563 */
5564GMMR0DECL(int) GMMR0FindDuplicatePageReq(PGVM pGVM,