/* $Id: GMMR0.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */ /** @file * GMM - Global Memory Manager. */ /* * Copyright (C) 2007-2023 Oracle and/or its affiliates. * * This file is part of VirtualBox base platform packages, as * available from https://www.virtualbox.org. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation, in version 3 of the * License. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, see . * * SPDX-License-Identifier: GPL-3.0-only */ /** @page pg_gmm GMM - The Global Memory Manager * * As the name indicates, this component is responsible for global memory * management. Currently only guest RAM is allocated from the GMM, but this * may change to include shadow page tables and other bits later. * * Guest RAM is managed as individual pages, but allocated from the host OS * in chunks for reasons of portability / efficiency. To minimize the memory * footprint all tracking structure must be as small as possible without * unnecessary performance penalties. * * The allocation chunks has fixed sized, the size defined at compile time * by the #GMM_CHUNK_SIZE \#define. * * Each chunk is given an unique ID. Each page also has a unique ID. The * relationship between the two IDs is: * @code * GMM_CHUNK_SHIFT = log2(GMM_CHUNK_SIZE / GUEST_PAGE_SIZE); * idPage = (idChunk << GMM_CHUNK_SHIFT) | iPage; * @endcode * Where iPage is the index of the page within the chunk. This ID scheme * permits for efficient chunk and page lookup, but it relies on the chunk size * to be set at compile time. The chunks are organized in an AVL tree with their * IDs being the keys. * * The physical address of each page in an allocation chunk is maintained by * the #RTR0MEMOBJ and obtained using #RTR0MemObjGetPagePhysAddr. There is no * need to duplicate this information (it'll cost 8-bytes per page if we did). * * So what do we need to track per page? Most importantly we need to know * which state the page is in: * - Private - Allocated for (eventually) backing one particular VM page. * - Shared - Readonly page that is used by one or more VMs and treated * as COW by PGM. * - Free - Not used by anyone. * * For the page replacement operations (sharing, defragmenting and freeing) * to be somewhat efficient, private pages needs to be associated with a * particular page in a particular VM. * * Tracking the usage of shared pages is impractical and expensive, so we'll * settle for a reference counting system instead. * * Free pages will be chained on LIFOs * * On 64-bit systems we will use a 64-bit bitfield per page, while on 32-bit * systems a 32-bit bitfield will have to suffice because of address space * limitations. The #GMMPAGE structure shows the details. * * * @section sec_gmm_alloc_strat Page Allocation Strategy * * The strategy for allocating pages has to take fragmentation and shared * pages into account, or we may end up with with 2000 chunks with only * a few pages in each. Shared pages cannot easily be reallocated because * of the inaccurate usage accounting (see above). Private pages can be * reallocated by a defragmentation thread in the same manner that sharing * is done. * * The first approach is to manage the free pages in two sets depending on * whether they are mainly for the allocation of shared or private pages. * In the initial implementation there will be almost no possibility for * mixing shared and private pages in the same chunk (only if we're really * stressed on memory), but when we implement forking of VMs and have to * deal with lots of COW pages it'll start getting kind of interesting. * * The sets are lists of chunks with approximately the same number of * free pages. Say the chunk size is 1MB, meaning 256 pages, and a set * consists of 16 lists. So, the first list will contain the chunks with * 1-7 free pages, the second covers 8-15, and so on. The chunks will be * moved between the lists as pages are freed up or allocated. * * * @section sec_gmm_costs Costs * * The per page cost in kernel space is 32-bit plus whatever RTR0MEMOBJ * entails. In addition there is the chunk cost of approximately * (sizeof(RT0MEMOBJ) + sizeof(CHUNK)) / 2^CHUNK_SHIFT bytes per page. * * On Windows the per page #RTR0MEMOBJ cost is 32-bit on 32-bit windows * and 64-bit on 64-bit windows (a PFN_NUMBER in the MDL). So, 64-bit per page. * The cost on Linux is identical, but here it's because of sizeof(struct page *). * * * @section sec_gmm_legacy Legacy Mode for Non-Tier-1 Platforms * * In legacy mode the page source is locked user pages and not * #RTR0MemObjAllocPhysNC, this means that a page can only be allocated * by the VM that locked it. We will make no attempt at implementing * page sharing on these systems, just do enough to make it all work. * * @note With 6.1 really dropping 32-bit support, the legacy mode is obsoleted * under the assumption that there is sufficient kernel virtual address * space to map all of the guest memory allocations. So, we'll be using * #RTR0MemObjAllocPage on some platforms as an alternative to * #RTR0MemObjAllocPhysNC. * * * @subsection sub_gmm_locking Serializing * * One simple fast mutex will be employed in the initial implementation, not * two as mentioned in @ref sec_pgmPhys_Serializing. * * @see @ref sec_pgmPhys_Serializing * * * @section sec_gmm_overcommit Memory Over-Commitment Management * * The GVM will have to do the system wide memory over-commitment * management. My current ideas are: * - Per VM oc policy that indicates how much to initially commit * to it and what to do in a out-of-memory situation. * - Prevent overtaxing the host. * * There are some challenges here, the main ones are configurability and * security. Should we for instance permit anyone to request 100% memory * commitment? Who should be allowed to do runtime adjustments of the * config. And how to prevent these settings from being lost when the last * VM process exits? The solution is probably to have an optional root * daemon the will keep VMMR0.r0 in memory and enable the security measures. * * * * @section sec_gmm_numa NUMA * * NUMA considerations will be designed and implemented a bit later. * * The preliminary guesses is that we will have to try allocate memory as * close as possible to the CPUs the VM is executed on (EMT and additional CPU * threads). Which means it's mostly about allocation and sharing policies. * Both the scheduler and allocator interface will to supply some NUMA info * and we'll need to have a way to calc access costs. * */ /********************************************************************************************************************************* * Header Files * *********************************************************************************************************************************/ #define LOG_GROUP LOG_GROUP_GMM #include #include #include "GMMR0Internal.h" #include #include #include #include #include #include #include #include #ifdef VBOX_STRICT # include #endif #include #include #include #include #include #include #include #include #include /* This is 64-bit only code now. */ #if HC_ARCH_BITS != 64 || ARCH_BITS != 64 # error "This is 64-bit only code" #endif /********************************************************************************************************************************* * Defined Constants And Macros * *********************************************************************************************************************************/ /** @def VBOX_USE_CRIT_SECT_FOR_GIANT * Use a critical section instead of a fast mutex for the giant GMM lock. * * @remarks This is primarily a way of avoiding the deadlock checks in the * windows driver verifier. */ #if defined(RT_OS_WINDOWS) || defined(RT_OS_DARWIN) || defined(DOXYGEN_RUNNING) # define VBOX_USE_CRIT_SECT_FOR_GIANT #endif /********************************************************************************************************************************* * Structures and Typedefs * *********************************************************************************************************************************/ /** Pointer to set of free chunks. */ typedef struct GMMCHUNKFREESET *PGMMCHUNKFREESET; /** * The per-page tracking structure employed by the GMM. * * Because of the different layout on 32-bit and 64-bit hosts in earlier * versions of the code, macros are used to get and set some of the data. */ typedef union GMMPAGE { /** Unsigned integer view. */ uint64_t u; /** The common view. */ struct GMMPAGECOMMON { uint32_t uStuff1 : 32; uint32_t uStuff2 : 30; /** The page state. */ uint32_t u2State : 2; } Common; /** The view of a private page. */ struct GMMPAGEPRIVATE { /** The guest page frame number. (Max addressable: 2 ^ 44 - 16) */ uint32_t pfn; /** The GVM handle. (64K VMs) */ uint32_t hGVM : 16; /** Reserved. */ uint32_t u16Reserved : 14; /** The page state. */ uint32_t u2State : 2; } Private; /** The view of a shared page. */ struct GMMPAGESHARED { /** The host page frame number. (Max addressable: 2 ^ 44 - 16) */ uint32_t pfn; /** The reference count (64K VMs). */ uint32_t cRefs : 16; /** Used for debug checksumming. */ uint32_t u14Checksum : 14; /** The page state. */ uint32_t u2State : 2; } Shared; /** The view of a free page. */ struct GMMPAGEFREE { /** The index of the next page in the free list. UINT16_MAX is NIL. */ uint16_t iNext; /** Reserved. Checksum or something? */ uint16_t u16Reserved0; /** Reserved. Checksum or something? */ uint32_t u30Reserved1 : 29; /** Set if the page was zeroed. */ uint32_t fZeroed : 1; /** The page state. */ uint32_t u2State : 2; } Free; } GMMPAGE; AssertCompileSize(GMMPAGE, sizeof(RTHCUINTPTR)); /** Pointer to a GMMPAGE. */ typedef GMMPAGE *PGMMPAGE; /** @name The Page States. * @{ */ /** A private page. */ #define GMM_PAGE_STATE_PRIVATE 0 /** A shared page. */ #define GMM_PAGE_STATE_SHARED 2 /** A free page. */ #define GMM_PAGE_STATE_FREE 3 /** @} */ /** @def GMM_PAGE_IS_PRIVATE * * @returns true if private, false if not. * @param pPage The GMM page. */ #define GMM_PAGE_IS_PRIVATE(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_PRIVATE ) /** @def GMM_PAGE_IS_SHARED * * @returns true if shared, false if not. * @param pPage The GMM page. */ #define GMM_PAGE_IS_SHARED(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_SHARED ) /** @def GMM_PAGE_IS_FREE * * @returns true if free, false if not. * @param pPage The GMM page. */ #define GMM_PAGE_IS_FREE(pPage) ( (pPage)->Common.u2State == GMM_PAGE_STATE_FREE ) /** @def GMM_PAGE_PFN_LAST * The last valid guest pfn range. * @remark Some of the values outside the range has special meaning, * see GMM_PAGE_PFN_UNSHAREABLE. */ #define GMM_PAGE_PFN_LAST UINT32_C(0xfffffff0) AssertCompile(GMM_PAGE_PFN_LAST == (GMM_GCPHYS_LAST >> GUEST_PAGE_SHIFT)); /** @def GMM_PAGE_PFN_UNSHAREABLE * Indicates that this page isn't used for normal guest memory and thus isn't shareable. */ #define GMM_PAGE_PFN_UNSHAREABLE UINT32_C(0xfffffff1) AssertCompile(GMM_PAGE_PFN_UNSHAREABLE == (GMM_GCPHYS_UNSHAREABLE >> GUEST_PAGE_SHIFT)); /** * A GMM allocation chunk ring-3 mapping record. * * This should really be associated with a session and not a VM, but * it's simpler to associated with a VM and cleanup with the VM object * is destroyed. */ typedef struct GMMCHUNKMAP { /** The mapping object. */ RTR0MEMOBJ hMapObj; /** The VM owning the mapping. */ PGVM pGVM; } GMMCHUNKMAP; /** Pointer to a GMM allocation chunk mapping. */ typedef struct GMMCHUNKMAP *PGMMCHUNKMAP; /** * A GMM allocation chunk. */ typedef struct GMMCHUNK { /** The AVL node core. * The Key is the chunk ID. (Giant mtx.) */ AVLU32NODECORE Core; /** The memory object. * Either from RTR0MemObjAllocPhysNC or RTR0MemObjLockUser depending on * what the host can dish up with. (Chunk mtx protects mapping accesses * and related frees.) */ RTR0MEMOBJ hMemObj; #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM /** Pointer to the kernel mapping. */ uint8_t *pbMapping; #endif /** Pointer to the next chunk in the free list. (Giant mtx.) */ PGMMCHUNK pFreeNext; /** Pointer to the previous chunk in the free list. (Giant mtx.) */ PGMMCHUNK pFreePrev; /** Pointer to the free set this chunk belongs to. NULL for * chunks with no free pages. (Giant mtx.) */ PGMMCHUNKFREESET pSet; /** List node in the chunk list (GMM::ChunkList). (Giant mtx.) */ RTLISTNODE ListNode; /** Pointer to an array of mappings. (Chunk mtx.) */ PGMMCHUNKMAP paMappingsX; /** The number of mappings. (Chunk mtx.) */ uint16_t cMappingsX; /** The mapping lock this chunk is using using. UINT8_MAX if nobody is mapping * or freeing anything. (Giant mtx.) */ uint8_t volatile iChunkMtx; /** GMM_CHUNK_FLAGS_XXX. (Giant mtx.) */ uint8_t fFlags; /** The head of the list of free pages. UINT16_MAX is the NIL value. * (Giant mtx.) */ uint16_t iFreeHead; /** The number of free pages. (Giant mtx.) */ uint16_t cFree; /** The GVM handle of the VM that first allocated pages from this chunk, this * is used as a preference when there are several chunks to choose from. * When in bound memory mode this isn't a preference any longer. (Giant * mtx.) */ uint16_t hGVM; /** The ID of the NUMA node the memory mostly resides on. (Reserved for * future use.) (Giant mtx.) */ uint16_t idNumaNode; /** The number of private pages. (Giant mtx.) */ uint16_t cPrivate; /** The number of shared pages. (Giant mtx.) */ uint16_t cShared; /** The UID this chunk is associated with. */ RTUID uidOwner; uint32_t u32Padding; /** The pages. (Giant mtx.) */ GMMPAGE aPages[GMM_CHUNK_NUM_PAGES]; } GMMCHUNK; /** Indicates that the NUMA properies of the memory is unknown. */ #define GMM_CHUNK_NUMA_ID_UNKNOWN UINT16_C(0xfffe) /** @name GMM_CHUNK_FLAGS_XXX - chunk flags. * @{ */ /** Indicates that the chunk is a large page (2MB). */ #define GMM_CHUNK_FLAGS_LARGE_PAGE UINT16_C(0x0001) /** @} */ /** * An allocation chunk TLB entry. */ typedef struct GMMCHUNKTLBE { /** The chunk id. */ uint32_t idChunk; /** Pointer to the chunk. */ PGMMCHUNK pChunk; } GMMCHUNKTLBE; /** Pointer to an allocation chunk TLB entry. */ typedef GMMCHUNKTLBE *PGMMCHUNKTLBE; /** The number of entries in the allocation chunk TLB. */ #define GMM_CHUNKTLB_ENTRIES 32 /** Gets the TLB entry index for the given Chunk ID. */ #define GMM_CHUNKTLB_IDX(idChunk) ( (idChunk) & (GMM_CHUNKTLB_ENTRIES - 1) ) /** * An allocation chunk TLB. */ typedef struct GMMCHUNKTLB { /** The TLB entries. */ GMMCHUNKTLBE aEntries[GMM_CHUNKTLB_ENTRIES]; } GMMCHUNKTLB; /** Pointer to an allocation chunk TLB. */ typedef GMMCHUNKTLB *PGMMCHUNKTLB; /** * The GMM instance data. */ typedef struct GMM { /** Magic / eye catcher. GMM_MAGIC */ uint32_t u32Magic; /** The number of threads waiting on the mutex. */ uint32_t cMtxContenders; #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT /** The critical section protecting the GMM. * More fine grained locking can be implemented later if necessary. */ RTCRITSECT GiantCritSect; #else /** The fast mutex protecting the GMM. * More fine grained locking can be implemented later if necessary. */ RTSEMFASTMUTEX hMtx; #endif #ifdef VBOX_STRICT /** The current mutex owner. */ RTNATIVETHREAD hMtxOwner; #endif /** Spinlock protecting the AVL tree. * @todo Make this a read-write spinlock as we should allow concurrent * lookups. */ RTSPINLOCK hSpinLockTree; /** The chunk tree. * Protected by hSpinLockTree. */ PAVLU32NODECORE pChunks; /** Chunk freeing generation - incremented whenever a chunk is freed. Used * for validating the per-VM chunk TLB entries. Valid range is 1 to 2^62 * (exclusive), though higher numbers may temporarily occure while * invalidating the individual TLBs during wrap-around processing. */ uint64_t volatile idFreeGeneration; /** The chunk TLB. * Protected by hSpinLockTree. */ GMMCHUNKTLB ChunkTLB; /** The private free set. */ GMMCHUNKFREESET PrivateX; /** The shared free set. */ GMMCHUNKFREESET Shared; /** Shared module tree (global). * @todo separate trees for distinctly different guest OSes. */ PAVLLU32NODECORE pGlobalSharedModuleTree; /** Sharable modules (count of nodes in pGlobalSharedModuleTree). */ uint32_t cShareableModules; /** The chunk list. For simplifying the cleanup process and avoid tree * traversal. */ RTLISTANCHOR ChunkList; /** The maximum number of pages we're allowed to allocate. * @gcfgm{GMM/MaxPages,64-bit, Direct.} * @gcfgm{GMM/PctPages,32-bit, Relative to the number of host pages.} */ uint64_t cMaxPages; /** The number of pages that has been reserved. * The deal is that cReservedPages - cOverCommittedPages <= cMaxPages. */ uint64_t cReservedPages; /** The number of pages that we have over-committed in reservations. */ uint64_t cOverCommittedPages; /** The number of actually allocated (committed if you like) pages. */ uint64_t cAllocatedPages; /** The number of pages that are shared. A subset of cAllocatedPages. */ uint64_t cSharedPages; /** The number of pages that are actually shared between VMs. */ uint64_t cDuplicatePages; /** The number of pages that are shared that has been left behind by * VMs not doing proper cleanups. */ uint64_t cLeftBehindSharedPages; /** The number of allocation chunks. * (The number of pages we've allocated from the host can be derived from this.) */ uint32_t cChunks; /** The number of current ballooned pages. */ uint64_t cBalloonedPages; #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM /** Whether #RTR0MemObjAllocPhysNC works. */ bool fHasWorkingAllocPhysNC; #else bool fPadding; #endif /** The bound memory mode indicator. * When set, the memory will be bound to a specific VM and never * shared. This is always set if fLegacyAllocationMode is set. * (Also determined at initialization time.) */ bool fBoundMemoryMode; /** The number of registered VMs. */ uint16_t cRegisteredVMs; /** The index of the next mutex to use. */ uint32_t iNextChunkMtx; /** Chunk locks for reducing lock contention without having to allocate * one lock per chunk. */ struct { /** The mutex */ RTSEMFASTMUTEX hMtx; /** The number of threads currently using this mutex. */ uint32_t volatile cUsers; } aChunkMtx[64]; /** The number of freed chunks ever. This is used as list generation to * avoid restarting the cleanup scanning when the list wasn't modified. */ uint32_t volatile cFreedChunks; /** The previous allocated Chunk ID. * Used as a hint to avoid scanning the whole bitmap. */ uint32_t idChunkPrev; /** Spinlock protecting idChunkPrev & bmChunkId. */ RTSPINLOCK hSpinLockChunkId; /** Chunk ID allocation bitmap. * Bits of allocated IDs are set, free ones are clear. * The NIL id (0) is marked allocated. */ uint32_t bmChunkId[(GMM_CHUNKID_LAST + 1 + 31) / 32]; } GMM; /** Pointer to the GMM instance. */ typedef GMM *PGMM; /** The value of GMM::u32Magic (Katsuhiro Otomo). */ #define GMM_MAGIC UINT32_C(0x19540414) /** * GMM chunk mutex state. * * This is returned by gmmR0ChunkMutexAcquire and is used by the other * gmmR0ChunkMutex* methods. */ typedef struct GMMR0CHUNKMTXSTATE { PGMM pGMM; /** The index of the chunk mutex. */ uint8_t iChunkMtx; /** The relevant flags (GMMR0CHUNK_MTX_XXX). */ uint8_t fFlags; } GMMR0CHUNKMTXSTATE; /** Pointer to a chunk mutex state. */ typedef GMMR0CHUNKMTXSTATE *PGMMR0CHUNKMTXSTATE; /** @name GMMR0CHUNK_MTX_XXX * @{ */ #define GMMR0CHUNK_MTX_INVALID UINT32_C(0) #define GMMR0CHUNK_MTX_KEEP_GIANT UINT32_C(1) #define GMMR0CHUNK_MTX_RETAKE_GIANT UINT32_C(2) #define GMMR0CHUNK_MTX_DROP_GIANT UINT32_C(3) #define GMMR0CHUNK_MTX_END UINT32_C(4) /** @} */ /** The maximum number of shared modules per-vm. */ #define GMM_MAX_SHARED_PER_VM_MODULES 2048 /** The maximum number of shared modules GMM is allowed to track. */ #define GMM_MAX_SHARED_GLOBAL_MODULES 16834 /** * Argument packet for gmmR0SharedModuleCleanup. */ typedef struct GMMR0SHMODPERVMDTORARGS { PGVM pGVM; PGMM pGMM; } GMMR0SHMODPERVMDTORARGS; /** * Argument packet for gmmR0CheckSharedModule. */ typedef struct GMMCHECKSHAREDMODULEINFO { PGVM pGVM; VMCPUID idCpu; } GMMCHECKSHAREDMODULEINFO; /********************************************************************************************************************************* * Global Variables * *********************************************************************************************************************************/ /** Pointer to the GMM instance data. */ static PGMM g_pGMM = NULL; /** Macro for obtaining and validating the g_pGMM pointer. * * On failure it will return from the invoking function with the specified * return value. * * @param pGMM The name of the pGMM variable. * @param rc The return value on failure. Use VERR_GMM_INSTANCE for VBox * status codes. */ #define GMM_GET_VALID_INSTANCE(pGMM, rc) \ do { \ (pGMM) = g_pGMM; \ AssertPtrReturn((pGMM), (rc)); \ AssertMsgReturn((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic), (rc)); \ } while (0) /** Macro for obtaining and validating the g_pGMM pointer, void function * variant. * * On failure it will return from the invoking function. * * @param pGMM The name of the pGMM variable. */ #define GMM_GET_VALID_INSTANCE_VOID(pGMM) \ do { \ (pGMM) = g_pGMM; \ AssertPtrReturnVoid((pGMM)); \ AssertMsgReturnVoid((pGMM)->u32Magic == GMM_MAGIC, ("%p - %#x\n", (pGMM), (pGMM)->u32Magic)); \ } while (0) /** @def GMM_CHECK_SANITY_UPON_ENTERING * Checks the sanity of the GMM instance data before making changes. * * This is macro is a stub by default and must be enabled manually in the code. * * @returns true if sane, false if not. * @param pGMM The name of the pGMM variable. */ #if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0 # define GMM_CHECK_SANITY_UPON_ENTERING(pGMM) (RT_LIKELY(gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0)) #else # define GMM_CHECK_SANITY_UPON_ENTERING(pGMM) (true) #endif /** @def GMM_CHECK_SANITY_UPON_LEAVING * Checks the sanity of the GMM instance data after making changes. * * This is macro is a stub by default and must be enabled manually in the code. * * @returns true if sane, false if not. * @param pGMM The name of the pGMM variable. */ #if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0 # define GMM_CHECK_SANITY_UPON_LEAVING(pGMM) (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0) #else # define GMM_CHECK_SANITY_UPON_LEAVING(pGMM) (true) #endif /** @def GMM_CHECK_SANITY_IN_LOOPS * Checks the sanity of the GMM instance in the allocation loops. * * This is macro is a stub by default and must be enabled manually in the code. * * @returns true if sane, false if not. * @param pGMM The name of the pGMM variable. */ #if defined(VBOX_STRICT) && defined(GMMR0_WITH_SANITY_CHECK) && 0 # define GMM_CHECK_SANITY_IN_LOOPS(pGMM) (gmmR0SanityCheck((pGMM), __PRETTY_FUNCTION__, __LINE__) == 0) #else # define GMM_CHECK_SANITY_IN_LOOPS(pGMM) (true) #endif /********************************************************************************************************************************* * Internal Functions * *********************************************************************************************************************************/ static DECLCALLBACK(int) gmmR0TermDestroyChunk(PAVLU32NODECORE pNode, void *pvGMM); static bool gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk); DECLINLINE(void) gmmR0UnlinkChunk(PGMMCHUNK pChunk); DECLINLINE(void) gmmR0LinkChunk(PGMMCHUNK pChunk, PGMMCHUNKFREESET pSet); DECLINLINE(void) gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk); #ifdef GMMR0_WITH_SANITY_CHECK static uint32_t gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo); #endif static bool gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem); DECLINLINE(void) gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage); DECLINLINE(void) gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage); static int gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk); #ifdef VBOX_WITH_PAGE_SHARING static void gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM); # ifdef VBOX_STRICT static uint32_t gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage); # endif #endif /** * Initializes the GMM component. * * This is called when the VMMR0.r0 module is loaded and protected by the * loader semaphore. * * @returns VBox status code. */ GMMR0DECL(int) GMMR0Init(void) { LogFlow(("GMMInit:\n")); /* Currently assuming same host and guest page size here. Can change it to dish out guest pages with different size from the host page later if needed, though a restriction would be the host page size must be larger than the guest page size. */ AssertCompile(GUEST_PAGE_SIZE == HOST_PAGE_SIZE); AssertCompile(GUEST_PAGE_SIZE <= HOST_PAGE_SIZE); /* * Allocate the instance data and the locks. */ PGMM pGMM = (PGMM)RTMemAllocZ(sizeof(*pGMM)); if (!pGMM) return VERR_NO_MEMORY; pGMM->u32Magic = GMM_MAGIC; for (unsigned i = 0; i < RT_ELEMENTS(pGMM->ChunkTLB.aEntries); i++) pGMM->ChunkTLB.aEntries[i].idChunk = NIL_GMM_CHUNKID; RTListInit(&pGMM->ChunkList); ASMBitSet(&pGMM->bmChunkId[0], NIL_GMM_CHUNKID); #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT int rc = RTCritSectInit(&pGMM->GiantCritSect); #else int rc = RTSemFastMutexCreate(&pGMM->hMtx); #endif if (RT_SUCCESS(rc)) { unsigned iMtx; for (iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++) { rc = RTSemFastMutexCreate(&pGMM->aChunkMtx[iMtx].hMtx); if (RT_FAILURE(rc)) break; } pGMM->hSpinLockTree = NIL_RTSPINLOCK; if (RT_SUCCESS(rc)) rc = RTSpinlockCreate(&pGMM->hSpinLockTree, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "gmm-chunk-tree"); pGMM->hSpinLockChunkId = NIL_RTSPINLOCK; if (RT_SUCCESS(rc)) rc = RTSpinlockCreate(&pGMM->hSpinLockChunkId, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "gmm-chunk-id"); if (RT_SUCCESS(rc)) { /* * Figure out how we're going to allocate stuff (only applicable to * host with linear physical memory mappings). */ pGMM->fBoundMemoryMode = false; #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM pGMM->fHasWorkingAllocPhysNC = false; RTR0MEMOBJ hMemObj; rc = RTR0MemObjAllocPhysNC(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS); if (RT_SUCCESS(rc)) { rc = RTR0MemObjFree(hMemObj, true); AssertRC(rc); pGMM->fHasWorkingAllocPhysNC = true; } else if (rc != VERR_NOT_SUPPORTED) SUPR0Printf("GMMR0Init: Warning! RTR0MemObjAllocPhysNC(, %u, NIL_RTHCPHYS) -> %d!\n", GMM_CHUNK_SIZE, rc); # endif /* * Query system page count and guess a reasonable cMaxPages value. */ pGMM->cMaxPages = UINT32_MAX; /** @todo IPRT function for query ram size and such. */ /* * The idFreeGeneration value should be set so we actually trigger the * wrap-around invalidation handling during a typical test run. */ pGMM->idFreeGeneration = UINT64_MAX / 4 - 128; g_pGMM = pGMM; #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM LogFlow(("GMMInit: pGMM=%p fBoundMemoryMode=%RTbool fHasWorkingAllocPhysNC=%RTbool\n", pGMM, pGMM->fBoundMemoryMode, pGMM->fHasWorkingAllocPhysNC)); #else LogFlow(("GMMInit: pGMM=%p fBoundMemoryMode=%RTbool\n", pGMM, pGMM->fBoundMemoryMode)); #endif return VINF_SUCCESS; } /* * Bail out. */ RTSpinlockDestroy(pGMM->hSpinLockChunkId); RTSpinlockDestroy(pGMM->hSpinLockTree); while (iMtx-- > 0) RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx); #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT RTCritSectDelete(&pGMM->GiantCritSect); #else RTSemFastMutexDestroy(pGMM->hMtx); #endif } pGMM->u32Magic = 0; RTMemFree(pGMM); SUPR0Printf("GMMR0Init: failed! rc=%d\n", rc); return rc; } /** * Terminates the GMM component. */ GMMR0DECL(void) GMMR0Term(void) { LogFlow(("GMMTerm:\n")); /* * Take care / be paranoid... */ PGMM pGMM = g_pGMM; if (!RT_VALID_PTR(pGMM)) return; if (pGMM->u32Magic != GMM_MAGIC) { SUPR0Printf("GMMR0Term: u32Magic=%#x\n", pGMM->u32Magic); return; } /* * Undo what init did and free all the resources we've acquired. */ /* Destroy the fundamentals. */ g_pGMM = NULL; pGMM->u32Magic = ~GMM_MAGIC; #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT RTCritSectDelete(&pGMM->GiantCritSect); #else RTSemFastMutexDestroy(pGMM->hMtx); pGMM->hMtx = NIL_RTSEMFASTMUTEX; #endif RTSpinlockDestroy(pGMM->hSpinLockTree); pGMM->hSpinLockTree = NIL_RTSPINLOCK; RTSpinlockDestroy(pGMM->hSpinLockChunkId); pGMM->hSpinLockChunkId = NIL_RTSPINLOCK; /* Free any chunks still hanging around. */ RTAvlU32Destroy(&pGMM->pChunks, gmmR0TermDestroyChunk, pGMM); /* Destroy the chunk locks. */ for (unsigned iMtx = 0; iMtx < RT_ELEMENTS(pGMM->aChunkMtx); iMtx++) { Assert(pGMM->aChunkMtx[iMtx].cUsers == 0); RTSemFastMutexDestroy(pGMM->aChunkMtx[iMtx].hMtx); pGMM->aChunkMtx[iMtx].hMtx = NIL_RTSEMFASTMUTEX; } /* Finally the instance data itself. */ RTMemFree(pGMM); LogFlow(("GMMTerm: done\n")); } /** * RTAvlU32Destroy callback. * * @returns 0 * @param pNode The node to destroy. * @param pvGMM The GMM handle. */ static DECLCALLBACK(int) gmmR0TermDestroyChunk(PAVLU32NODECORE pNode, void *pvGMM) { PGMMCHUNK pChunk = (PGMMCHUNK)pNode; if (pChunk->cFree != GMM_CHUNK_NUM_PAGES) SUPR0Printf("GMMR0Term: %RKv/%#x: cFree=%d cPrivate=%d cShared=%d cMappings=%d\n", pChunk, pChunk->Core.Key, pChunk->cFree, pChunk->cPrivate, pChunk->cShared, pChunk->cMappingsX); int rc = RTR0MemObjFree(pChunk->hMemObj, true /* fFreeMappings */); if (RT_FAILURE(rc)) { SUPR0Printf("GMMR0Term: %RKv/%#x: RTRMemObjFree(%RKv,true) -> %d (cMappings=%d)\n", pChunk, pChunk->Core.Key, pChunk->hMemObj, rc, pChunk->cMappingsX); AssertRC(rc); } pChunk->hMemObj = NIL_RTR0MEMOBJ; RTMemFree(pChunk->paMappingsX); pChunk->paMappingsX = NULL; RTMemFree(pChunk); NOREF(pvGMM); return 0; } /** * Initializes the per-VM data for the GMM. * * This is called from within the GVMM lock (from GVMMR0CreateVM) * and should only initialize the data members so GMMR0CleanupVM * can deal with them. We reserve no memory or anything here, * that's done later in GMMR0InitVM. * * @param pGVM Pointer to the Global VM structure. */ GMMR0DECL(int) GMMR0InitPerVMData(PGVM pGVM) { AssertCompile(RT_SIZEOFMEMB(GVM,gmm.s) <= RT_SIZEOFMEMB(GVM,gmm.padding)); pGVM->gmm.s.Stats.enmPolicy = GMMOCPOLICY_INVALID; pGVM->gmm.s.Stats.enmPriority = GMMPRIORITY_INVALID; pGVM->gmm.s.Stats.fMayAllocate = false; pGVM->gmm.s.hChunkTlbSpinLock = NIL_RTSPINLOCK; int rc = RTSpinlockCreate(&pGVM->gmm.s.hChunkTlbSpinLock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "per-vm-chunk-tlb"); AssertRCReturn(rc, rc); return VINF_SUCCESS; } /** * Acquires the GMM giant lock. * * @returns Assert status code from RTSemFastMutexRequest. * @param pGMM Pointer to the GMM instance. */ static int gmmR0MutexAcquire(PGMM pGMM) { ASMAtomicIncU32(&pGMM->cMtxContenders); #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT int rc = RTCritSectEnter(&pGMM->GiantCritSect); #else int rc = RTSemFastMutexRequest(pGMM->hMtx); #endif ASMAtomicDecU32(&pGMM->cMtxContenders); AssertRC(rc); #ifdef VBOX_STRICT pGMM->hMtxOwner = RTThreadNativeSelf(); #endif return rc; } /** * Releases the GMM giant lock. * * @returns Assert status code from RTSemFastMutexRequest. * @param pGMM Pointer to the GMM instance. */ static int gmmR0MutexRelease(PGMM pGMM) { #ifdef VBOX_STRICT pGMM->hMtxOwner = NIL_RTNATIVETHREAD; #endif #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT int rc = RTCritSectLeave(&pGMM->GiantCritSect); #else int rc = RTSemFastMutexRelease(pGMM->hMtx); AssertRC(rc); #endif return rc; } /** * Yields the GMM giant lock if there is contention and a certain minimum time * has elapsed since we took it. * * @returns @c true if the mutex was yielded, @c false if not. * @param pGMM Pointer to the GMM instance. * @param puLockNanoTS Where the lock acquisition time stamp is kept * (in/out). */ static bool gmmR0MutexYield(PGMM pGMM, uint64_t *puLockNanoTS) { /* * If nobody is contending the mutex, don't bother checking the time. */ if (ASMAtomicReadU32(&pGMM->cMtxContenders) == 0) return false; /* * Don't yield if we haven't executed for at least 2 milliseconds. */ uint64_t uNanoNow = RTTimeSystemNanoTS(); if (uNanoNow - *puLockNanoTS < UINT32_C(2000000)) return false; /* * Yield the mutex. */ #ifdef VBOX_STRICT pGMM->hMtxOwner = NIL_RTNATIVETHREAD; #endif ASMAtomicIncU32(&pGMM->cMtxContenders); #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT int rc1 = RTCritSectLeave(&pGMM->GiantCritSect); AssertRC(rc1); #else int rc1 = RTSemFastMutexRelease(pGMM->hMtx); AssertRC(rc1); #endif RTThreadYield(); #ifdef VBOX_USE_CRIT_SECT_FOR_GIANT int rc2 = RTCritSectEnter(&pGMM->GiantCritSect); AssertRC(rc2); #else int rc2 = RTSemFastMutexRequest(pGMM->hMtx); AssertRC(rc2); #endif *puLockNanoTS = RTTimeSystemNanoTS(); ASMAtomicDecU32(&pGMM->cMtxContenders); #ifdef VBOX_STRICT pGMM->hMtxOwner = RTThreadNativeSelf(); #endif return true; } /** * Acquires a chunk lock. * * The caller must own the giant lock. * * @returns Assert status code from RTSemFastMutexRequest. * @param pMtxState The chunk mutex state info. (Avoids * passing the same flags and stuff around * for subsequent release and drop-giant * calls.) * @param pGMM Pointer to the GMM instance. * @param pChunk Pointer to the chunk. * @param fFlags Flags regarding the giant lock, GMMR0CHUNK_MTX_XXX. */ static int gmmR0ChunkMutexAcquire(PGMMR0CHUNKMTXSTATE pMtxState, PGMM pGMM, PGMMCHUNK pChunk, uint32_t fFlags) { Assert(fFlags > GMMR0CHUNK_MTX_INVALID && fFlags < GMMR0CHUNK_MTX_END); Assert(pGMM->hMtxOwner == RTThreadNativeSelf()); pMtxState->pGMM = pGMM; pMtxState->fFlags = (uint8_t)fFlags; /* * Get the lock index and reference the lock. */ Assert(pGMM->hMtxOwner == RTThreadNativeSelf()); uint32_t iChunkMtx = pChunk->iChunkMtx; if (iChunkMtx == UINT8_MAX) { iChunkMtx = pGMM->iNextChunkMtx++; iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx); /* Try get an unused one... */ if (pGMM->aChunkMtx[iChunkMtx].cUsers) { iChunkMtx = pGMM->iNextChunkMtx++; iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx); if (pGMM->aChunkMtx[iChunkMtx].cUsers) { iChunkMtx = pGMM->iNextChunkMtx++; iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx); if (pGMM->aChunkMtx[iChunkMtx].cUsers) { iChunkMtx = pGMM->iNextChunkMtx++; iChunkMtx %= RT_ELEMENTS(pGMM->aChunkMtx); } } } pChunk->iChunkMtx = iChunkMtx; } AssertCompile(RT_ELEMENTS(pGMM->aChunkMtx) < UINT8_MAX); pMtxState->iChunkMtx = (uint8_t)iChunkMtx; ASMAtomicIncU32(&pGMM->aChunkMtx[iChunkMtx].cUsers); /* * Drop the giant? */ if (fFlags != GMMR0CHUNK_MTX_KEEP_GIANT) { /** @todo GMM life cycle cleanup (we may race someone * destroying and cleaning up GMM)? */ gmmR0MutexRelease(pGMM); } /* * Take the chunk mutex. */ int rc = RTSemFastMutexRequest(pGMM->aChunkMtx[iChunkMtx].hMtx); AssertRC(rc); return rc; } /** * Releases the GMM giant lock. * * @returns Assert status code from RTSemFastMutexRequest. * @param pMtxState Pointer to the chunk mutex state. * @param pChunk Pointer to the chunk if it's still * alive, NULL if it isn't. This is used to deassociate * the chunk from the mutex on the way out so a new one * can be selected next time, thus avoiding contented * mutexes. */ static int gmmR0ChunkMutexRelease(PGMMR0CHUNKMTXSTATE pMtxState, PGMMCHUNK pChunk) { PGMM pGMM = pMtxState->pGMM; /* * Release the chunk mutex and reacquire the giant if requested. */ int rc = RTSemFastMutexRelease(pGMM->aChunkMtx[pMtxState->iChunkMtx].hMtx); AssertRC(rc); if (pMtxState->fFlags == GMMR0CHUNK_MTX_RETAKE_GIANT) rc = gmmR0MutexAcquire(pGMM); else Assert((pMtxState->fFlags != GMMR0CHUNK_MTX_DROP_GIANT) == (pGMM->hMtxOwner == RTThreadNativeSelf())); /* * Drop the chunk mutex user reference and deassociate it from the chunk * when possible. */ if ( ASMAtomicDecU32(&pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers) == 0 && pChunk && RT_SUCCESS(rc) ) { if (pMtxState->fFlags != GMMR0CHUNK_MTX_DROP_GIANT) pChunk->iChunkMtx = UINT8_MAX; else { rc = gmmR0MutexAcquire(pGMM); if (RT_SUCCESS(rc)) { if (pGMM->aChunkMtx[pMtxState->iChunkMtx].cUsers == 0) pChunk->iChunkMtx = UINT8_MAX; rc = gmmR0MutexRelease(pGMM); } } } pMtxState->pGMM = NULL; return rc; } /** * Drops the giant GMM lock we kept in gmmR0ChunkMutexAcquire while keeping the * chunk locked. * * This only works if gmmR0ChunkMutexAcquire was called with * GMMR0CHUNK_MTX_KEEP_GIANT. gmmR0ChunkMutexRelease will retake the giant * mutex, i.e. behave as if GMMR0CHUNK_MTX_RETAKE_GIANT was used. * * @returns VBox status code (assuming success is ok). * @param pMtxState Pointer to the chunk mutex state. */ static int gmmR0ChunkMutexDropGiant(PGMMR0CHUNKMTXSTATE pMtxState) { AssertReturn(pMtxState->fFlags == GMMR0CHUNK_MTX_KEEP_GIANT, VERR_GMM_MTX_FLAGS); Assert(pMtxState->pGMM->hMtxOwner == RTThreadNativeSelf()); pMtxState->fFlags = GMMR0CHUNK_MTX_RETAKE_GIANT; /** @todo GMM life cycle cleanup (we may race someone * destroying and cleaning up GMM)? */ return gmmR0MutexRelease(pMtxState->pGMM); } /** * For experimenting with NUMA affinity and such. * * @returns The current NUMA Node ID. */ static uint16_t gmmR0GetCurrentNumaNodeId(void) { #if 1 return GMM_CHUNK_NUMA_ID_UNKNOWN; #else return RTMpCpuId() / 16; #endif } /** * Cleans up when a VM is terminating. * * @param pGVM Pointer to the Global VM structure. */ GMMR0DECL(void) GMMR0CleanupVM(PGVM pGVM) { LogFlow(("GMMR0CleanupVM: pGVM=%p:{.hSelf=%#x}\n", pGVM, pGVM->hSelf)); PGMM pGMM; GMM_GET_VALID_INSTANCE_VOID(pGMM); #ifdef VBOX_WITH_PAGE_SHARING /* * Clean up all registered shared modules first. */ gmmR0SharedModuleCleanup(pGMM, pGVM); #endif gmmR0MutexAcquire(pGMM); uint64_t uLockNanoTS = RTTimeSystemNanoTS(); GMM_CHECK_SANITY_UPON_ENTERING(pGMM); /* * The policy is 'INVALID' until the initial reservation * request has been serviced. */ if ( pGVM->gmm.s.Stats.enmPolicy > GMMOCPOLICY_INVALID && pGVM->gmm.s.Stats.enmPolicy < GMMOCPOLICY_END) { /* * If it's the last VM around, we can skip walking all the chunk looking * for the pages owned by this VM and instead flush the whole shebang. * * This takes care of the eventuality that a VM has left shared page * references behind (shouldn't happen of course, but you never know). */ Assert(pGMM->cRegisteredVMs); pGMM->cRegisteredVMs--; /* * Walk the entire pool looking for pages that belong to this VM * and leftover mappings. (This'll only catch private pages, * shared pages will be 'left behind'.) */ /** @todo r=bird: This scanning+freeing could be optimized in bound mode! */ uint64_t cPrivatePages = pGVM->gmm.s.Stats.cPrivatePages; /* save */ unsigned iCountDown = 64; bool fRedoFromStart; PGMMCHUNK pChunk; do { fRedoFromStart = false; RTListForEachReverse(&pGMM->ChunkList, pChunk, GMMCHUNK, ListNode) { uint32_t const cFreeChunksOld = pGMM->cFreedChunks; if ( ( !pGMM->fBoundMemoryMode || pChunk->hGVM == pGVM->hSelf) && gmmR0CleanupVMScanChunk(pGMM, pGVM, pChunk)) { /* We left the giant mutex, so reset the yield counters. */ uLockNanoTS = RTTimeSystemNanoTS(); iCountDown = 64; } else { /* Didn't leave it, so do normal yielding. */ if (!iCountDown) gmmR0MutexYield(pGMM, &uLockNanoTS); else iCountDown--; } if (pGMM->cFreedChunks != cFreeChunksOld) { fRedoFromStart = true; break; } } } while (fRedoFromStart); if (pGVM->gmm.s.Stats.cPrivatePages) SUPR0Printf("GMMR0CleanupVM: hGVM=%#x has %#x private pages that cannot be found!\n", pGVM->hSelf, pGVM->gmm.s.Stats.cPrivatePages); pGMM->cAllocatedPages -= cPrivatePages; /* * Free empty chunks. */ PGMMCHUNKFREESET pPrivateSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX; do { fRedoFromStart = false; iCountDown = 10240; pChunk = pPrivateSet->apLists[GMM_CHUNK_FREE_SET_UNUSED_LIST]; while (pChunk) { PGMMCHUNK pNext = pChunk->pFreeNext; Assert(pChunk->cFree == GMM_CHUNK_NUM_PAGES); if ( !pGMM->fBoundMemoryMode || pChunk->hGVM == pGVM->hSelf) { uint64_t const idGenerationOld = pPrivateSet->idGeneration; if (gmmR0FreeChunk(pGMM, pGVM, pChunk, true /*fRelaxedSem*/)) { /* We've left the giant mutex, restart? (+1 for our unlink) */ fRedoFromStart = pPrivateSet->idGeneration != idGenerationOld + 1; if (fRedoFromStart) break; uLockNanoTS = RTTimeSystemNanoTS(); iCountDown = 10240; } } /* Advance and maybe yield the lock. */ pChunk = pNext; if (--iCountDown == 0) { uint64_t const idGenerationOld = pPrivateSet->idGeneration; fRedoFromStart = gmmR0MutexYield(pGMM, &uLockNanoTS) && pPrivateSet->idGeneration != idGenerationOld; if (fRedoFromStart) break; iCountDown = 10240; } } } while (fRedoFromStart); /* * Account for shared pages that weren't freed. */ if (pGVM->gmm.s.Stats.cSharedPages) { Assert(pGMM->cSharedPages >= pGVM->gmm.s.Stats.cSharedPages); SUPR0Printf("GMMR0CleanupVM: hGVM=%#x left %#x shared pages behind!\n", pGVM->hSelf, pGVM->gmm.s.Stats.cSharedPages); pGMM->cLeftBehindSharedPages += pGVM->gmm.s.Stats.cSharedPages; } /* * Clean up balloon statistics in case the VM process crashed. */ Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages); pGMM->cBalloonedPages -= pGVM->gmm.s.Stats.cBalloonedPages; /* * Update the over-commitment management statistics. */ pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages + pGVM->gmm.s.Stats.Reserved.cFixedPages + pGVM->gmm.s.Stats.Reserved.cShadowPages; switch (pGVM->gmm.s.Stats.enmPolicy) { case GMMOCPOLICY_NO_OC: break; default: /** @todo Update GMM->cOverCommittedPages */ break; } } /* zap the GVM data. */ pGVM->gmm.s.Stats.enmPolicy = GMMOCPOLICY_INVALID; pGVM->gmm.s.Stats.enmPriority = GMMPRIORITY_INVALID; pGVM->gmm.s.Stats.fMayAllocate = false; GMM_CHECK_SANITY_UPON_LEAVING(pGMM); gmmR0MutexRelease(pGMM); /* * Destroy the spinlock. */ RTSPINLOCK hSpinlock = NIL_RTSPINLOCK; ASMAtomicXchgHandle(&pGVM->gmm.s.hChunkTlbSpinLock, NIL_RTSPINLOCK, &hSpinlock); RTSpinlockDestroy(hSpinlock); LogFlow(("GMMR0CleanupVM: returns\n")); } /** * Scan one chunk for private pages belonging to the specified VM. * * @note This function may drop the giant mutex! * * @returns @c true if we've temporarily dropped the giant mutex, @c false if * we didn't. * @param pGMM Pointer to the GMM instance. * @param pGVM The global VM handle. * @param pChunk The chunk to scan. */ static bool gmmR0CleanupVMScanChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk) { Assert(!pGMM->fBoundMemoryMode || pChunk->hGVM == pGVM->hSelf); /* * Look for pages belonging to the VM. * (Perform some internal checks while we're scanning.) */ #ifndef VBOX_STRICT if (pChunk->cFree != GMM_CHUNK_NUM_PAGES) #endif { unsigned cPrivate = 0; unsigned cShared = 0; unsigned cFree = 0; gmmR0UnlinkChunk(pChunk); /* avoiding cFreePages updates. */ uint16_t hGVM = pGVM->hSelf; unsigned iPage = (GMM_CHUNK_SIZE >> GUEST_PAGE_SHIFT); while (iPage-- > 0) if (GMM_PAGE_IS_PRIVATE(&pChunk->aPages[iPage])) { if (pChunk->aPages[iPage].Private.hGVM == hGVM) { /* * Free the page. * * The reason for not using gmmR0FreePrivatePage here is that we * must *not* cause the chunk to be freed from under us - we're in * an AVL tree walk here. */ pChunk->aPages[iPage].u = 0; pChunk->aPages[iPage].Free.u2State = GMM_PAGE_STATE_FREE; pChunk->aPages[iPage].Free.fZeroed = false; pChunk->aPages[iPage].Free.iNext = pChunk->iFreeHead; pChunk->iFreeHead = iPage; pChunk->cPrivate--; pChunk->cFree++; pGVM->gmm.s.Stats.cPrivatePages--; cFree++; } else cPrivate++; } else if (GMM_PAGE_IS_FREE(&pChunk->aPages[iPage])) cFree++; else cShared++; gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk); /* * Did it add up? */ if (RT_UNLIKELY( pChunk->cFree != cFree || pChunk->cPrivate != cPrivate || pChunk->cShared != cShared)) { SUPR0Printf("gmmR0CleanupVMScanChunk: Chunk %RKv/%#x has bogus stats - free=%d/%d private=%d/%d shared=%d/%d\n", pChunk, pChunk->Core.Key, pChunk->cFree, cFree, pChunk->cPrivate, cPrivate, pChunk->cShared, cShared); pChunk->cFree = cFree; pChunk->cPrivate = cPrivate; pChunk->cShared = cShared; } } /* * If not in bound memory mode, we should reset the hGVM field * if it has our handle in it. */ if (pChunk->hGVM == pGVM->hSelf) { if (!g_pGMM->fBoundMemoryMode) pChunk->hGVM = NIL_GVM_HANDLE; else if (pChunk->cFree != GMM_CHUNK_NUM_PAGES) { SUPR0Printf("gmmR0CleanupVMScanChunk: %RKv/%#x: cFree=%#x - it should be 0 in bound mode!\n", pChunk, pChunk->Core.Key, pChunk->cFree); AssertMsgFailed(("%p/%#x: cFree=%#x - it should be 0 in bound mode!\n", pChunk, pChunk->Core.Key, pChunk->cFree)); gmmR0UnlinkChunk(pChunk); pChunk->cFree = GMM_CHUNK_NUM_PAGES; gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk); } } /* * Look for a mapping belonging to the terminating VM. */ GMMR0CHUNKMTXSTATE MtxState; gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT); unsigned cMappings = pChunk->cMappingsX; for (unsigned i = 0; i < cMappings; i++) if (pChunk->paMappingsX[i].pGVM == pGVM) { gmmR0ChunkMutexDropGiant(&MtxState); RTR0MEMOBJ hMemObj = pChunk->paMappingsX[i].hMapObj; cMappings--; if (i < cMappings) pChunk->paMappingsX[i] = pChunk->paMappingsX[cMappings]; pChunk->paMappingsX[cMappings].pGVM = NULL; pChunk->paMappingsX[cMappings].hMapObj = NIL_RTR0MEMOBJ; Assert(pChunk->cMappingsX - 1U == cMappings); pChunk->cMappingsX = cMappings; int rc = RTR0MemObjFree(hMemObj, false /* fFreeMappings (NA) */); if (RT_FAILURE(rc)) { SUPR0Printf("gmmR0CleanupVMScanChunk: %RKv/%#x: mapping #%x: RTRMemObjFree(%RKv,false) -> %d \n", pChunk, pChunk->Core.Key, i, hMemObj, rc); AssertRC(rc); } gmmR0ChunkMutexRelease(&MtxState, pChunk); return true; } gmmR0ChunkMutexRelease(&MtxState, pChunk); return false; } /** * The initial resource reservations. * * This will make memory reservations according to policy and priority. If there aren't * sufficient resources available to sustain the VM this function will fail and all * future allocations requests will fail as well. * * These are just the initial reservations made very very early during the VM creation * process and will be adjusted later in the GMMR0UpdateReservation call after the * ring-3 init has completed. * * @returns VBox status code. * @retval VERR_GMM_MEMORY_RESERVATION_DECLINED * @retval VERR_GMM_ * * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id - must be zero. * @param cBasePages The number of pages that may be allocated for the base RAM and ROMs. * This does not include MMIO2 and similar. * @param cShadowPages The number of pages that may be allocated for shadow paging structures. * @param cFixedPages The number of pages that may be allocated for fixed objects like the * hyper heap, MMIO2 and similar. * @param enmPolicy The OC policy to use on this VM. * @param enmPriority The priority in an out-of-memory situation. * * @thread The creator thread / EMT(0). */ GMMR0DECL(int) GMMR0InitialReservation(PGVM pGVM, VMCPUID idCpu, uint64_t cBasePages, uint32_t cShadowPages, uint32_t cFixedPages, GMMOCPOLICY enmPolicy, GMMPRIORITY enmPriority) { LogFlow(("GMMR0InitialReservation: pGVM=%p cBasePages=%#llx cShadowPages=%#x cFixedPages=%#x enmPolicy=%d enmPriority=%d\n", pGVM, cBasePages, cShadowPages, cFixedPages, enmPolicy, enmPriority)); /* * Validate, get basics and take the semaphore. */ AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID); PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; AssertReturn(cBasePages, VERR_INVALID_PARAMETER); AssertReturn(cShadowPages, VERR_INVALID_PARAMETER); AssertReturn(cFixedPages, VERR_INVALID_PARAMETER); AssertReturn(enmPolicy > GMMOCPOLICY_INVALID && enmPolicy < GMMOCPOLICY_END, VERR_INVALID_PARAMETER); AssertReturn(enmPriority > GMMPRIORITY_INVALID && enmPriority < GMMPRIORITY_END, VERR_INVALID_PARAMETER); gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { if ( !pGVM->gmm.s.Stats.Reserved.cBasePages && !pGVM->gmm.s.Stats.Reserved.cFixedPages && !pGVM->gmm.s.Stats.Reserved.cShadowPages) { /* * Check if we can accommodate this. */ /* ... later ... */ if (RT_SUCCESS(rc)) { /* * Update the records. */ pGVM->gmm.s.Stats.Reserved.cBasePages = cBasePages; pGVM->gmm.s.Stats.Reserved.cFixedPages = cFixedPages; pGVM->gmm.s.Stats.Reserved.cShadowPages = cShadowPages; pGVM->gmm.s.Stats.enmPolicy = enmPolicy; pGVM->gmm.s.Stats.enmPriority = enmPriority; pGVM->gmm.s.Stats.fMayAllocate = true; pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages; pGMM->cRegisteredVMs++; } } else rc = VERR_WRONG_ORDER; GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); LogFlow(("GMMR0InitialReservation: returns %Rrc\n", rc)); return rc; } /** * VMMR0 request wrapper for GMMR0InitialReservation. * * @returns see GMMR0InitialReservation. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0InitialReservationReq(PGVM pGVM, VMCPUID idCpu, PGMMINITIALRESERVATIONREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pGVM, VERR_INVALID_POINTER); AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER); return GMMR0InitialReservation(pGVM, idCpu, pReq->cBasePages, pReq->cShadowPages, pReq->cFixedPages, pReq->enmPolicy, pReq->enmPriority); } /** * This updates the memory reservation with the additional MMIO2 and ROM pages. * * @returns VBox status code. * @retval VERR_GMM_MEMORY_RESERVATION_DECLINED * * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param cBasePages The number of pages that may be allocated for the base RAM and ROMs. * This does not include MMIO2 and similar. * @param cShadowPages The number of pages that may be allocated for shadow paging structures. * @param cFixedPages The number of pages that may be allocated for fixed objects like the * hyper heap, MMIO2 and similar. * * @thread EMT(idCpu) */ GMMR0DECL(int) GMMR0UpdateReservation(PGVM pGVM, VMCPUID idCpu, uint64_t cBasePages, uint32_t cShadowPages, uint32_t cFixedPages) { LogFlow(("GMMR0UpdateReservation: pGVM=%p cBasePages=%#llx cShadowPages=%#x cFixedPages=%#x\n", pGVM, cBasePages, cShadowPages, cFixedPages)); /* * Validate, get basics and take the semaphore. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; AssertReturn(cBasePages, VERR_INVALID_PARAMETER); AssertReturn(cShadowPages, VERR_INVALID_PARAMETER); AssertReturn(cFixedPages, VERR_INVALID_PARAMETER); gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { if ( pGVM->gmm.s.Stats.Reserved.cBasePages && pGVM->gmm.s.Stats.Reserved.cFixedPages && pGVM->gmm.s.Stats.Reserved.cShadowPages) { /* * Check if we can accommodate this. */ /* ... later ... */ if (RT_SUCCESS(rc)) { /* * Update the records. */ pGMM->cReservedPages -= pGVM->gmm.s.Stats.Reserved.cBasePages + pGVM->gmm.s.Stats.Reserved.cFixedPages + pGVM->gmm.s.Stats.Reserved.cShadowPages; pGMM->cReservedPages += cBasePages + cFixedPages + cShadowPages; pGVM->gmm.s.Stats.Reserved.cBasePages = cBasePages; pGVM->gmm.s.Stats.Reserved.cFixedPages = cFixedPages; pGVM->gmm.s.Stats.Reserved.cShadowPages = cShadowPages; } } else rc = VERR_WRONG_ORDER; GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); LogFlow(("GMMR0UpdateReservation: returns %Rrc\n", rc)); return rc; } /** * VMMR0 request wrapper for GMMR0UpdateReservation. * * @returns see GMMR0UpdateReservation. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0UpdateReservationReq(PGVM pGVM, VMCPUID idCpu, PGMMUPDATERESERVATIONREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER); return GMMR0UpdateReservation(pGVM, idCpu, pReq->cBasePages, pReq->cShadowPages, pReq->cFixedPages); } #ifdef GMMR0_WITH_SANITY_CHECK /** * Performs sanity checks on a free set. * * @returns Error count. * * @param pGMM Pointer to the GMM instance. * @param pSet Pointer to the set. * @param pszSetName The set name. * @param pszFunction The function from which it was called. * @param uLine The line number. */ static uint32_t gmmR0SanityCheckSet(PGMM pGMM, PGMMCHUNKFREESET pSet, const char *pszSetName, const char *pszFunction, unsigned uLineNo) { uint32_t cErrors = 0; /* * Count the free pages in all the chunks and match it against pSet->cFreePages. */ uint32_t cPages = 0; for (unsigned i = 0; i < RT_ELEMENTS(pSet->apLists); i++) { for (PGMMCHUNK pCur = pSet->apLists[i]; pCur; pCur = pCur->pFreeNext) { /** @todo check that the chunk is hash into the right set. */ cPages += pCur->cFree; } } if (RT_UNLIKELY(cPages != pSet->cFreePages)) { SUPR0Printf("GMM insanity: found %#x pages in the %s set, expected %#x. (%s, line %u)\n", cPages, pszSetName, pSet->cFreePages, pszFunction, uLineNo); cErrors++; } return cErrors; } /** * Performs some sanity checks on the GMM while owning lock. * * @returns Error count. * * @param pGMM Pointer to the GMM instance. * @param pszFunction The function from which it is called. * @param uLineNo The line number. */ static uint32_t gmmR0SanityCheck(PGMM pGMM, const char *pszFunction, unsigned uLineNo) { uint32_t cErrors = 0; cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->PrivateX, "private", pszFunction, uLineNo); cErrors += gmmR0SanityCheckSet(pGMM, &pGMM->Shared, "shared", pszFunction, uLineNo); /** @todo add more sanity checks. */ return cErrors; } #endif /* GMMR0_WITH_SANITY_CHECK */ /** * Looks up a chunk in the tree and fill in the TLB entry for it. * * This is not expected to fail and will bitch if it does. * * @returns Pointer to the allocation chunk, NULL if not found. * @param pGMM Pointer to the GMM instance. * @param idChunk The ID of the chunk to find. * @param pTlbe Pointer to the TLB entry. * * @note Caller owns spinlock. */ static PGMMCHUNK gmmR0GetChunkSlow(PGMM pGMM, uint32_t idChunk, PGMMCHUNKTLBE pTlbe) { PGMMCHUNK pChunk = (PGMMCHUNK)RTAvlU32Get(&pGMM->pChunks, idChunk); AssertMsgReturn(pChunk, ("Chunk %#x not found!\n", idChunk), NULL); pTlbe->idChunk = idChunk; pTlbe->pChunk = pChunk; return pChunk; } /** * Finds a allocation chunk, spin-locked. * * This is not expected to fail and will bitch if it does. * * @returns Pointer to the allocation chunk, NULL if not found. * @param pGMM Pointer to the GMM instance. * @param idChunk The ID of the chunk to find. */ DECLINLINE(PGMMCHUNK) gmmR0GetChunkLocked(PGMM pGMM, uint32_t idChunk) { /* * Do a TLB lookup, branch if not in the TLB. */ PGMMCHUNKTLBE pTlbe = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(idChunk)]; PGMMCHUNK pChunk = pTlbe->pChunk; if ( pChunk == NULL || pTlbe->idChunk != idChunk) pChunk = gmmR0GetChunkSlow(pGMM, idChunk, pTlbe); return pChunk; } /** * Finds a allocation chunk. * * This is not expected to fail and will bitch if it does. * * @returns Pointer to the allocation chunk, NULL if not found. * @param pGMM Pointer to the GMM instance. * @param idChunk The ID of the chunk to find. */ DECLINLINE(PGMMCHUNK) gmmR0GetChunk(PGMM pGMM, uint32_t idChunk) { RTSpinlockAcquire(pGMM->hSpinLockTree); PGMMCHUNK pChunk = gmmR0GetChunkLocked(pGMM, idChunk); RTSpinlockRelease(pGMM->hSpinLockTree); return pChunk; } /** * Finds a page. * * This is not expected to fail and will bitch if it does. * * @returns Pointer to the page, NULL if not found. * @param pGMM Pointer to the GMM instance. * @param idPage The ID of the page to find. */ DECLINLINE(PGMMPAGE) gmmR0GetPage(PGMM pGMM, uint32_t idPage) { PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT); if (RT_LIKELY(pChunk)) return &pChunk->aPages[idPage & GMM_PAGEID_IDX_MASK]; return NULL; } #if 0 /* unused */ /** * Gets the host physical address for a page given by it's ID. * * @returns The host physical address or NIL_RTHCPHYS. * @param pGMM Pointer to the GMM instance. * @param idPage The ID of the page to find. */ DECLINLINE(RTHCPHYS) gmmR0GetPageHCPhys(PGMM pGMM, uint32_t idPage) { PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT); if (RT_LIKELY(pChunk)) return RTR0MemObjGetPagePhysAddr(pChunk->hMemObj, idPage & GMM_PAGEID_IDX_MASK); return NIL_RTHCPHYS; } #endif /* unused */ /** * Selects the appropriate free list given the number of free pages. * * @returns Free list index. * @param cFree The number of free pages in the chunk. */ DECLINLINE(unsigned) gmmR0SelectFreeSetList(unsigned cFree) { unsigned iList = cFree >> GMM_CHUNK_FREE_SET_SHIFT; AssertMsg(iList < RT_SIZEOFMEMB(GMMCHUNKFREESET, apLists) / RT_SIZEOFMEMB(GMMCHUNKFREESET, apLists[0]), ("%d (%u)\n", iList, cFree)); return iList; } /** * Unlinks the chunk from the free list it's currently on (if any). * * @param pChunk The allocation chunk. */ DECLINLINE(void) gmmR0UnlinkChunk(PGMMCHUNK pChunk) { PGMMCHUNKFREESET pSet = pChunk->pSet; if (RT_LIKELY(pSet)) { pSet->cFreePages -= pChunk->cFree; pSet->idGeneration++; PGMMCHUNK pPrev = pChunk->pFreePrev; PGMMCHUNK pNext = pChunk->pFreeNext; if (pPrev) pPrev->pFreeNext = pNext; else pSet->apLists[gmmR0SelectFreeSetList(pChunk->cFree)] = pNext; if (pNext) pNext->pFreePrev = pPrev; pChunk->pSet = NULL; pChunk->pFreeNext = NULL; pChunk->pFreePrev = NULL; } else { Assert(!pChunk->pFreeNext); Assert(!pChunk->pFreePrev); Assert(!pChunk->cFree); } } /** * Links the chunk onto the appropriate free list in the specified free set. * * If no free entries, it's not linked into any list. * * @param pChunk The allocation chunk. * @param pSet The free set. */ DECLINLINE(void) gmmR0LinkChunk(PGMMCHUNK pChunk, PGMMCHUNKFREESET pSet) { Assert(!pChunk->pSet); Assert(!pChunk->pFreeNext); Assert(!pChunk->pFreePrev); if (pChunk->cFree > 0) { pChunk->pSet = pSet; pChunk->pFreePrev = NULL; unsigned const iList = gmmR0SelectFreeSetList(pChunk->cFree); pChunk->pFreeNext = pSet->apLists[iList]; if (pChunk->pFreeNext) pChunk->pFreeNext->pFreePrev = pChunk; pSet->apLists[iList] = pChunk; pSet->cFreePages += pChunk->cFree; pSet->idGeneration++; } } /** * Links the chunk onto the appropriate free list in the specified free set. * * If no free entries, it's not linked into any list. * * @param pGMM Pointer to the GMM instance. * @param pGVM Pointer to the kernel-only VM instace data. * @param pChunk The allocation chunk. */ DECLINLINE(void) gmmR0SelectSetAndLinkChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk) { PGMMCHUNKFREESET pSet; if (pGMM->fBoundMemoryMode) pSet = &pGVM->gmm.s.Private; else if (pChunk->cShared) pSet = &pGMM->Shared; else pSet = &pGMM->PrivateX; gmmR0LinkChunk(pChunk, pSet); } /** * Frees a Chunk ID. * * @param pGMM Pointer to the GMM instance. * @param idChunk The Chunk ID to free. */ static void gmmR0FreeChunkId(PGMM pGMM, uint32_t idChunk) { AssertReturnVoid(idChunk != NIL_GMM_CHUNKID); RTSpinlockAcquire(pGMM->hSpinLockChunkId); /* We could probably skip the locking here, I think. */ AssertMsg(ASMBitTest(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk)); ASMAtomicBitClear(&pGMM->bmChunkId[0], idChunk); RTSpinlockRelease(pGMM->hSpinLockChunkId); } /** * Allocates a new Chunk ID. * * @returns The Chunk ID. * @param pGMM Pointer to the GMM instance. */ static uint32_t gmmR0AllocateChunkId(PGMM pGMM) { AssertCompile(!((GMM_CHUNKID_LAST + 1) & 31)); /* must be a multiple of 32 */ AssertCompile(NIL_GMM_CHUNKID == 0); RTSpinlockAcquire(pGMM->hSpinLockChunkId); /* * Try the next sequential one. */ int32_t idChunk = ++pGMM->idChunkPrev; if ( (uint32_t)idChunk <= GMM_CHUNKID_LAST && idChunk > NIL_GMM_CHUNKID) { if (!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk)) { RTSpinlockRelease(pGMM->hSpinLockChunkId); return idChunk; } /* * Scan sequentially from the last one. */ if ((uint32_t)idChunk < GMM_CHUNKID_LAST) { idChunk = ASMBitNextClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1, idChunk); if ( idChunk > NIL_GMM_CHUNKID && (uint32_t)idChunk <= GMM_CHUNKID_LAST) { AssertMsgReturnStmt(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk), RTSpinlockRelease(pGMM->hSpinLockChunkId), NIL_GMM_CHUNKID); pGMM->idChunkPrev = idChunk; RTSpinlockRelease(pGMM->hSpinLockChunkId); return idChunk; } } } /* * Ok, scan from the start. * We're not racing anyone, so there is no need to expect failures or have restart loops. */ idChunk = ASMBitFirstClear(&pGMM->bmChunkId[0], GMM_CHUNKID_LAST + 1); AssertMsgReturnStmt(idChunk > NIL_GMM_CHUNKID && (uint32_t)idChunk <= GMM_CHUNKID_LAST, ("%#x\n", idChunk), RTSpinlockRelease(pGMM->hSpinLockChunkId), NIL_GVM_HANDLE); AssertMsgReturnStmt(!ASMAtomicBitTestAndSet(&pGMM->bmChunkId[0], idChunk), ("%#x\n", idChunk), RTSpinlockRelease(pGMM->hSpinLockChunkId), NIL_GMM_CHUNKID); pGMM->idChunkPrev = idChunk; RTSpinlockRelease(pGMM->hSpinLockChunkId); return idChunk; } /** * Allocates one private page. * * Worker for gmmR0AllocatePages. * * @param pChunk The chunk to allocate it from. * @param hGVM The GVM handle of the VM requesting memory. * @param pPageDesc The page descriptor. */ static void gmmR0AllocatePage(PGMMCHUNK pChunk, uint32_t hGVM, PGMMPAGEDESC pPageDesc) { /* update the chunk stats. */ if (pChunk->hGVM == NIL_GVM_HANDLE) pChunk->hGVM = hGVM; Assert(pChunk->cFree); pChunk->cFree--; pChunk->cPrivate++; /* unlink the first free page. */ const uint32_t iPage = pChunk->iFreeHead; AssertReleaseMsg(iPage < RT_ELEMENTS(pChunk->aPages), ("%d\n", iPage)); PGMMPAGE pPage = &pChunk->aPages[iPage]; Assert(GMM_PAGE_IS_FREE(pPage)); pChunk->iFreeHead = pPage->Free.iNext; Log3(("A pPage=%p iPage=%#x/%#x u2State=%d iFreeHead=%#x iNext=%#x\n", pPage, iPage, (pChunk->Core.Key << GMM_CHUNKID_SHIFT) | iPage, pPage->Common.u2State, pChunk->iFreeHead, pPage->Free.iNext)); bool const fZeroed = pPage->Free.fZeroed; /* make the page private. */ pPage->u = 0; AssertCompile(GMM_PAGE_STATE_PRIVATE == 0); pPage->Private.hGVM = hGVM; AssertCompile(NIL_RTHCPHYS >= GMM_GCPHYS_LAST); AssertCompile(GMM_GCPHYS_UNSHAREABLE >= GMM_GCPHYS_LAST); if (pPageDesc->HCPhysGCPhys <= GMM_GCPHYS_LAST) pPage->Private.pfn = pPageDesc->HCPhysGCPhys >> GUEST_PAGE_SHIFT; else pPage->Private.pfn = GMM_PAGE_PFN_UNSHAREABLE; /* unshareable / unassigned - same thing. */ /* update the page descriptor. */ pPageDesc->idSharedPage = NIL_GMM_PAGEID; pPageDesc->idPage = (pChunk->Core.Key << GMM_CHUNKID_SHIFT) | iPage; RTHCPHYS const HCPhys = RTR0MemObjGetPagePhysAddr(pChunk->hMemObj, iPage); Assert(HCPhys != NIL_RTHCPHYS); Assert(HCPhys < NIL_GMMPAGEDESC_PHYS); pPageDesc->HCPhysGCPhys = HCPhys; pPageDesc->fZeroed = fZeroed; } /** * Picks the free pages from a chunk. * * @returns The new page descriptor table index. * @param pChunk The chunk. * @param hGVM The affinity of the chunk. NIL_GVM_HANDLE for no * affinity. * @param iPage The current page descriptor table index. * @param cPages The total number of pages to allocate. * @param paPages The page descriptor table (input + ouput). */ static uint32_t gmmR0AllocatePagesFromChunk(PGMMCHUNK pChunk, uint16_t const hGVM, uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages) { PGMMCHUNKFREESET pSet = pChunk->pSet; Assert(pSet); gmmR0UnlinkChunk(pChunk); for (; pChunk->cFree && iPage < cPages; iPage++) gmmR0AllocatePage(pChunk, hGVM, &paPages[iPage]); gmmR0LinkChunk(pChunk, pSet); return iPage; } /** * Registers a new chunk of memory. * * This is called by gmmR0AllocateOneChunk and GMMR0AllocateLargePage. * * In the GMMR0AllocateLargePage case the GMM_CHUNK_FLAGS_LARGE_PAGE flag is * set and the chunk will be registered as fully allocated to save time. * * @returns VBox status code. On success, the giant GMM lock will be held, the * caller must release it (ugly). * @param pGMM Pointer to the GMM instance. * @param pSet Pointer to the set. * @param hMemObj The memory object for the chunk. * @param hGVM The affinity of the chunk. NIL_GVM_HANDLE for no * affinity. * @param pSession Same as @a hGVM. * @param fChunkFlags The chunk flags, GMM_CHUNK_FLAGS_XXX. * @param cPages The number of pages requested. Zero for large pages. * @param paPages The page descriptor table (input + output). NULL for * large pages. * @param piPage The pointer to the page descriptor table index variable. * This will be updated. NULL for large pages. * @param ppChunk Chunk address (out). * * @remarks The caller must not own the giant GMM mutex. * The giant GMM mutex will be acquired and returned acquired in * the success path. On failure, no locks will be held. */ static int gmmR0RegisterChunk(PGMM pGMM, PGMMCHUNKFREESET pSet, RTR0MEMOBJ hMemObj, uint16_t hGVM, PSUPDRVSESSION pSession, uint16_t fChunkFlags, uint32_t cPages, PGMMPAGEDESC paPages, uint32_t *piPage, PGMMCHUNK *ppChunk) { /* * Validate input & state. */ Assert(pGMM->hMtxOwner != RTThreadNativeSelf()); Assert(hGVM != NIL_GVM_HANDLE || pGMM->fBoundMemoryMode); Assert(fChunkFlags == 0 || fChunkFlags == GMM_CHUNK_FLAGS_LARGE_PAGE); if (!(fChunkFlags &= GMM_CHUNK_FLAGS_LARGE_PAGE)) { AssertPtr(paPages); AssertPtr(piPage); Assert(cPages > 0); Assert(cPages > *piPage); } else { Assert(cPages == 0); Assert(!paPages); Assert(!piPage); } #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM /* * Get a ring-0 mapping of the object. */ uint8_t *pbMapping = (uint8_t *)RTR0MemObjAddress(hMemObj); if (!pbMapping) { RTR0MEMOBJ hMapObj; int rc = RTR0MemObjMapKernel(&hMapObj, hMemObj, (void *)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE); if (RT_SUCCESS(rc)) pbMapping = (uint8_t *)RTR0MemObjAddress(hMapObj); else return rc; AssertPtr(pbMapping); } #endif /* * Allocate a chunk and an ID for it. */ int rc; PGMMCHUNK pChunk = (PGMMCHUNK)RTMemAllocZ(sizeof(*pChunk)); if (pChunk) { pChunk->Core.Key = gmmR0AllocateChunkId(pGMM); if ( pChunk->Core.Key != NIL_GMM_CHUNKID && pChunk->Core.Key <= GMM_CHUNKID_LAST) { /* * Initialize it. */ pChunk->hMemObj = hMemObj; #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM pChunk->pbMapping = pbMapping; #endif pChunk->hGVM = hGVM; pChunk->idNumaNode = gmmR0GetCurrentNumaNodeId(); pChunk->iChunkMtx = UINT8_MAX; pChunk->fFlags = fChunkFlags; pChunk->uidOwner = pSession ? SUPR0GetSessionUid(pSession) : NIL_RTUID; /*pChunk->cShared = 0; */ uint32_t const iDstPageFirst = piPage ? *piPage : cPages; if (!(fChunkFlags & GMM_CHUNK_FLAGS_LARGE_PAGE)) { /* * Allocate the requested number of pages from the start of the chunk, * queue the rest (if any) on the free list. */ uint32_t const cPagesAlloc = RT_MIN(cPages - iDstPageFirst, GMM_CHUNK_NUM_PAGES); pChunk->cPrivate = cPagesAlloc; pChunk->cFree = GMM_CHUNK_NUM_PAGES - cPagesAlloc; pChunk->iFreeHead = GMM_CHUNK_NUM_PAGES > cPagesAlloc ? cPagesAlloc : UINT16_MAX; /* Alloc pages: */ uint32_t const idPageChunk = pChunk->Core.Key << GMM_CHUNKID_SHIFT; uint32_t iDstPage = iDstPageFirst; uint32_t iPage; for (iPage = 0; iPage < cPagesAlloc; iPage++, iDstPage++) { if (paPages[iDstPage].HCPhysGCPhys <= GMM_GCPHYS_LAST) pChunk->aPages[iPage].Private.pfn = paPages[iDstPage].HCPhysGCPhys >> GUEST_PAGE_SHIFT; else pChunk->aPages[iPage].Private.pfn = GMM_PAGE_PFN_UNSHAREABLE; /* unshareable / unassigned - same thing. */ pChunk->aPages[iPage].Private.hGVM = hGVM; pChunk->aPages[iPage].Private.u2State = GMM_PAGE_STATE_PRIVATE; paPages[iDstPage].HCPhysGCPhys = RTR0MemObjGetPagePhysAddr(hMemObj, iPage); paPages[iDstPage].fZeroed = true; paPages[iDstPage].idPage = idPageChunk | iPage; paPages[iDstPage].idSharedPage = NIL_GMM_PAGEID; } *piPage = iDstPage; /* Build free list: */ if (iPage < RT_ELEMENTS(pChunk->aPages)) { Assert(pChunk->iFreeHead == iPage); for (; iPage < RT_ELEMENTS(pChunk->aPages) - 1; iPage++) { pChunk->aPages[iPage].Free.u2State = GMM_PAGE_STATE_FREE; pChunk->aPages[iPage].Free.fZeroed = true; pChunk->aPages[iPage].Free.iNext = iPage + 1; } pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.u2State = GMM_PAGE_STATE_FREE; pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.fZeroed = true; pChunk->aPages[RT_ELEMENTS(pChunk->aPages) - 1].Free.iNext = UINT16_MAX; } else Assert(pChunk->iFreeHead == UINT16_MAX); } else { /* * Large page: Mark all pages as privately allocated (watered down gmmR0AllocatePage). */ pChunk->cFree = 0; pChunk->cPrivate = GMM_CHUNK_NUM_PAGES; pChunk->iFreeHead = UINT16_MAX; for (unsigned iPage = 0; iPage < RT_ELEMENTS(pChunk->aPages); iPage++) { pChunk->aPages[iPage].Private.pfn = GMM_PAGE_PFN_UNSHAREABLE; pChunk->aPages[iPage].Private.hGVM = hGVM; pChunk->aPages[iPage].Private.u2State = GMM_PAGE_STATE_PRIVATE; } } /* * Zero the memory if it wasn't zeroed by the host already. * This simplifies keeping secret kernel bits from userland and brings * everyone to the same level wrt allocation zeroing. */ rc = VINF_SUCCESS; if (!RTR0MemObjWasZeroInitialized(hMemObj)) { #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM if (!(fChunkFlags & GMM_CHUNK_FLAGS_LARGE_PAGE)) { for (uint32_t iPage = 0; iPage < GMM_CHUNK_SIZE / HOST_PAGE_SIZE; iPage++) { void *pvPage = NULL; rc = SUPR0HCPhysToVirt(RTR0MemObjGetPagePhysAddr(hMemObj, iPage), &pvPage); AssertRCBreak(rc); RT_BZERO(pvPage, HOST_PAGE_SIZE); } } else { /* Can do the whole large page in one go. */ void *pvPage = NULL; rc = SUPR0HCPhysToVirt(RTR0MemObjGetPagePhysAddr(hMemObj, 0), &pvPage); AssertRC(rc); if (RT_SUCCESS(rc)) RT_BZERO(pvPage, GMM_CHUNK_SIZE); } #else RT_BZERO(pbMapping, GMM_CHUNK_SIZE); #endif } if (RT_SUCCESS(rc)) { *ppChunk = pChunk; /* * Allocate a Chunk ID and insert it into the tree. * This has to be done behind the mutex of course. */ rc = gmmR0MutexAcquire(pGMM); if (RT_SUCCESS(rc)) { if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { RTSpinlockAcquire(pGMM->hSpinLockTree); if (RTAvlU32Insert(&pGMM->pChunks, &pChunk->Core)) { pGMM->cChunks++; RTListAppend(&pGMM->ChunkList, &pChunk->ListNode); RTSpinlockRelease(pGMM->hSpinLockTree); gmmR0LinkChunk(pChunk, pSet); LogFlow(("gmmR0RegisterChunk: pChunk=%p id=%#x cChunks=%d\n", pChunk, pChunk->Core.Key, pGMM->cChunks)); GMM_CHECK_SANITY_UPON_LEAVING(pGMM); return VINF_SUCCESS; } /* * Bail out. */ RTSpinlockRelease(pGMM->hSpinLockTree); rc = VERR_GMM_CHUNK_INSERT; } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); } *ppChunk = NULL; } /* Undo any page allocations. */ if (!(fChunkFlags & GMM_CHUNK_FLAGS_LARGE_PAGE)) { uint32_t const cToFree = pChunk->cPrivate; Assert(*piPage - iDstPageFirst == cToFree); for (uint32_t iDstPage = iDstPageFirst, iPage = 0; iPage < cToFree; iPage++, iDstPage++) { paPages[iDstPageFirst].fZeroed = false; if (pChunk->aPages[iPage].Private.pfn == GMM_PAGE_PFN_UNSHAREABLE) paPages[iDstPageFirst].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS; else paPages[iDstPageFirst].HCPhysGCPhys = (RTHCPHYS)pChunk->aPages[iPage].Private.pfn << GUEST_PAGE_SHIFT; paPages[iDstPageFirst].idPage = NIL_GMM_PAGEID; paPages[iDstPageFirst].idSharedPage = NIL_GMM_PAGEID; } *piPage = iDstPageFirst; } gmmR0FreeChunkId(pGMM, pChunk->Core.Key); } else rc = VERR_GMM_CHUNK_INSERT; RTMemFree(pChunk); } else rc = VERR_NO_MEMORY; return rc; } /** * Allocate a new chunk, immediately pick the requested pages from it, and adds * what's remaining to the specified free set. * * @note This will leave the giant mutex while allocating the new chunk! * * @returns VBox status code. * @param pGMM Pointer to the GMM instance data. * @param pGVM Pointer to the kernel-only VM instace data. * @param pSet Pointer to the free set. * @param cPages The number of pages requested. * @param paPages The page descriptor table (input + output). * @param piPage The pointer to the page descriptor table index variable. * This will be updated. */ static int gmmR0AllocateChunkNew(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet, uint32_t cPages, PGMMPAGEDESC paPages, uint32_t *piPage) { gmmR0MutexRelease(pGMM); RTR0MEMOBJ hMemObj; int rc; #ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM if (pGMM->fHasWorkingAllocPhysNC) rc = RTR0MemObjAllocPhysNC(&hMemObj, GMM_CHUNK_SIZE, NIL_RTHCPHYS); else #endif rc = RTR0MemObjAllocPage(&hMemObj, GMM_CHUNK_SIZE, false /*fExecutable*/); if (RT_SUCCESS(rc)) { PGMMCHUNK pIgnored; rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, pGVM->pSession, 0 /*fChunkFlags*/, cPages, paPages, piPage, &pIgnored); if (RT_SUCCESS(rc)) return VINF_SUCCESS; /* bail out */ RTR0MemObjFree(hMemObj, true /* fFreeMappings */); } int rc2 = gmmR0MutexAcquire(pGMM); AssertRCReturn(rc2, RT_FAILURE(rc) ? rc : rc2); return rc; } /** * As a last restort we'll pick any page we can get. * * @returns The new page descriptor table index. * @param pSet The set to pick from. * @param pGVM Pointer to the global VM structure. * @param uidSelf The UID of the caller. * @param iPage The current page descriptor table index. * @param cPages The total number of pages to allocate. * @param paPages The page descriptor table (input + ouput). */ static uint32_t gmmR0AllocatePagesIndiscriminately(PGMMCHUNKFREESET pSet, PGVM pGVM, RTUID uidSelf, uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages) { unsigned iList = RT_ELEMENTS(pSet->apLists); while (iList-- > 0) { PGMMCHUNK pChunk = pSet->apLists[iList]; while (pChunk) { PGMMCHUNK pNext = pChunk->pFreeNext; if ( pChunk->uidOwner == uidSelf || ( pChunk->cMappingsX == 0 && pChunk->cFree == (GMM_CHUNK_SIZE >> GUEST_PAGE_SHIFT))) { iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages); if (iPage >= cPages) return iPage; } pChunk = pNext; } } return iPage; } /** * Pick pages from empty chunks on the same NUMA node. * * @returns The new page descriptor table index. * @param pSet The set to pick from. * @param pGVM Pointer to the global VM structure. * @param uidSelf The UID of the caller. * @param iPage The current page descriptor table index. * @param cPages The total number of pages to allocate. * @param paPages The page descriptor table (input + ouput). */ static uint32_t gmmR0AllocatePagesFromEmptyChunksOnSameNode(PGMMCHUNKFREESET pSet, PGVM pGVM, RTUID uidSelf, uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages) { PGMMCHUNK pChunk = pSet->apLists[GMM_CHUNK_FREE_SET_UNUSED_LIST]; if (pChunk) { uint16_t const idNumaNode = gmmR0GetCurrentNumaNodeId(); while (pChunk) { PGMMCHUNK pNext = pChunk->pFreeNext; if ( pChunk->idNumaNode == idNumaNode && ( pChunk->uidOwner == uidSelf || pChunk->cMappingsX == 0)) { pChunk->hGVM = pGVM->hSelf; pChunk->uidOwner = uidSelf; iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages); if (iPage >= cPages) { pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID; return iPage; } } pChunk = pNext; } } return iPage; } /** * Pick pages from non-empty chunks on the same NUMA node. * * @returns The new page descriptor table index. * @param pSet The set to pick from. * @param pGVM Pointer to the global VM structure. * @param uidSelf The UID of the caller. * @param iPage The current page descriptor table index. * @param cPages The total number of pages to allocate. * @param paPages The page descriptor table (input + ouput). */ static uint32_t gmmR0AllocatePagesFromSameNode(PGMMCHUNKFREESET pSet, PGVM pGVM, RTUID const uidSelf, uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages) { /** @todo start by picking from chunks with about the right size first? */ uint16_t const idNumaNode = gmmR0GetCurrentNumaNodeId(); unsigned iList = GMM_CHUNK_FREE_SET_UNUSED_LIST; while (iList-- > 0) { PGMMCHUNK pChunk = pSet->apLists[iList]; while (pChunk) { PGMMCHUNK pNext = pChunk->pFreeNext; if ( pChunk->idNumaNode == idNumaNode && pChunk->uidOwner == uidSelf) { iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages); if (iPage >= cPages) { pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID; return iPage; } } pChunk = pNext; } } return iPage; } /** * Pick pages that are in chunks already associated with the VM. * * @returns The new page descriptor table index. * @param pGMM Pointer to the GMM instance data. * @param pGVM Pointer to the global VM structure. * @param pSet The set to pick from. * @param iPage The current page descriptor table index. * @param cPages The total number of pages to allocate. * @param paPages The page descriptor table (input + ouput). */ static uint32_t gmmR0AllocatePagesAssociatedWithVM(PGMM pGMM, PGVM pGVM, PGMMCHUNKFREESET pSet, uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages) { uint16_t const hGVM = pGVM->hSelf; /* Hint. */ if (pGVM->gmm.s.idLastChunkHint != NIL_GMM_CHUNKID) { PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pGVM->gmm.s.idLastChunkHint); if (pChunk && pChunk->cFree) { iPage = gmmR0AllocatePagesFromChunk(pChunk, hGVM, iPage, cPages, paPages); if (iPage >= cPages) return iPage; } } /* Scan. */ for (unsigned iList = 0; iList < RT_ELEMENTS(pSet->apLists); iList++) { PGMMCHUNK pChunk = pSet->apLists[iList]; while (pChunk) { PGMMCHUNK pNext = pChunk->pFreeNext; if (pChunk->hGVM == hGVM) { iPage = gmmR0AllocatePagesFromChunk(pChunk, hGVM, iPage, cPages, paPages); if (iPage >= cPages) { pGVM->gmm.s.idLastChunkHint = pChunk->cFree ? pChunk->Core.Key : NIL_GMM_CHUNKID; return iPage; } } pChunk = pNext; } } return iPage; } /** * Pick pages in bound memory mode. * * @returns The new page descriptor table index. * @param pGVM Pointer to the global VM structure. * @param iPage The current page descriptor table index. * @param cPages The total number of pages to allocate. * @param paPages The page descriptor table (input + ouput). */ static uint32_t gmmR0AllocatePagesInBoundMode(PGVM pGVM, uint32_t iPage, uint32_t cPages, PGMMPAGEDESC paPages) { for (unsigned iList = 0; iList < RT_ELEMENTS(pGVM->gmm.s.Private.apLists); iList++) { PGMMCHUNK pChunk = pGVM->gmm.s.Private.apLists[iList]; while (pChunk) { Assert(pChunk->hGVM == pGVM->hSelf); PGMMCHUNK pNext = pChunk->pFreeNext; iPage = gmmR0AllocatePagesFromChunk(pChunk, pGVM->hSelf, iPage, cPages, paPages); if (iPage >= cPages) return iPage; pChunk = pNext; } } return iPage; } /** * Checks if we should start picking pages from chunks of other VMs because * we're getting close to the system memory or reserved limit. * * @returns @c true if we should, @c false if we should first try allocate more * chunks. */ static bool gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLimits(PGVM pGVM) { /* * Don't allocate a new chunk if we're */ uint64_t cPgReserved = pGVM->gmm.s.Stats.Reserved.cBasePages + pGVM->gmm.s.Stats.Reserved.cFixedPages - pGVM->gmm.s.Stats.cBalloonedPages /** @todo what about shared pages? */; uint64_t cPgAllocated = pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.Allocated.cFixedPages; uint64_t cPgDelta = cPgReserved - cPgAllocated; if (cPgDelta < GMM_CHUNK_NUM_PAGES * 4) return true; /** @todo make the threshold configurable, also test the code to see if * this ever kicks in (we might be reserving too much or smth). */ /* * Check how close we're to the max memory limit and how many fragments * there are?... */ /** @todo */ return false; } /** * Checks if we should start picking pages from chunks of other VMs because * there is a lot of free pages around. * * @returns @c true if we should, @c false if we should first try allocate more * chunks. */ static bool gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(PGMM pGMM) { /* * Setting the limit at 16 chunks (32 MB) at the moment. */ if (pGMM->PrivateX.cFreePages >= GMM_CHUNK_NUM_PAGES * 16) return true; return false; } /** * Common worker for GMMR0AllocateHandyPages and GMMR0AllocatePages. * * @returns VBox status code: * @retval VINF_SUCCESS on success. * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages. * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit, * that is we're trying to allocate more than we've reserved. * * @param pGMM Pointer to the GMM instance data. * @param pGVM Pointer to the VM. * @param cPages The number of pages to allocate. * @param paPages Pointer to the page descriptors. See GMMPAGEDESC for * details on what is expected on input. * @param enmAccount The account to charge. * * @remarks Caller owns the giant GMM lock. */ static int gmmR0AllocatePagesNew(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMPAGEDESC paPages, GMMACCOUNT enmAccount) { Assert(pGMM->hMtxOwner == RTThreadNativeSelf()); /* * Check allocation limits. */ if (RT_LIKELY(pGMM->cAllocatedPages + cPages <= pGMM->cMaxPages)) { /* likely */ } else return VERR_GMM_HIT_GLOBAL_LIMIT; switch (enmAccount) { case GMMACCOUNT_BASE: if (RT_LIKELY( pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + cPages <= pGVM->gmm.s.Stats.Reserved.cBasePages)) { /* likely */ } else { Log(("gmmR0AllocatePages:Base: Reserved=%#llx Allocated+Ballooned+Requested=%#llx+%#llx+%#x!\n", pGVM->gmm.s.Stats.Reserved.cBasePages, pGVM->gmm.s.Stats.Allocated.cBasePages, pGVM->gmm.s.Stats.cBalloonedPages, cPages)); return VERR_GMM_HIT_VM_ACCOUNT_LIMIT; } break; case GMMACCOUNT_SHADOW: if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cShadowPages + cPages <= pGVM->gmm.s.Stats.Reserved.cShadowPages)) { /* likely */ } else { Log(("gmmR0AllocatePages:Shadow: Reserved=%#x Allocated+Requested=%#x+%#x!\n", pGVM->gmm.s.Stats.Reserved.cShadowPages, pGVM->gmm.s.Stats.Allocated.cShadowPages, cPages)); return VERR_GMM_HIT_VM_ACCOUNT_LIMIT; } break; case GMMACCOUNT_FIXED: if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cFixedPages + cPages <= pGVM->gmm.s.Stats.Reserved.cFixedPages)) { /* likely */ } else { Log(("gmmR0AllocatePages:Fixed: Reserved=%#x Allocated+Requested=%#x+%#x!\n", pGVM->gmm.s.Stats.Reserved.cFixedPages, pGVM->gmm.s.Stats.Allocated.cFixedPages, cPages)); return VERR_GMM_HIT_VM_ACCOUNT_LIMIT; } break; default: AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE); } /* * Update the accounts before we proceed because we might be leaving the * protection of the global mutex and thus run the risk of permitting * too much memory to be allocated. */ switch (enmAccount) { case GMMACCOUNT_BASE: pGVM->gmm.s.Stats.Allocated.cBasePages += cPages; break; case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages += cPages; break; case GMMACCOUNT_FIXED: pGVM->gmm.s.Stats.Allocated.cFixedPages += cPages; break; default: AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE); } pGVM->gmm.s.Stats.cPrivatePages += cPages; pGMM->cAllocatedPages += cPages; /* * Bound mode is also relatively straightforward. */ uint32_t iPage = 0; int rc = VINF_SUCCESS; if (pGMM->fBoundMemoryMode) { iPage = gmmR0AllocatePagesInBoundMode(pGVM, iPage, cPages, paPages); if (iPage < cPages) do rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGVM->gmm.s.Private, cPages, paPages, &iPage); while (iPage < cPages && RT_SUCCESS(rc)); } /* * Shared mode is trickier as we should try archive the same locality as * in bound mode, but smartly make use of non-full chunks allocated by * other VMs if we're low on memory. */ else { RTUID const uidSelf = SUPR0GetSessionUid(pGVM->pSession); /* Pick the most optimal pages first. */ iPage = gmmR0AllocatePagesAssociatedWithVM(pGMM, pGVM, &pGMM->PrivateX, iPage, cPages, paPages); if (iPage < cPages) { /* Maybe we should try getting pages from chunks "belonging" to other VMs before allocating more chunks? */ bool fTriedOnSameAlready = false; if (gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLimits(pGVM)) { iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages); fTriedOnSameAlready = true; } /* Allocate memory from empty chunks. */ if (iPage < cPages) iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages); /* Grab empty shared chunks. */ if (iPage < cPages) iPage = gmmR0AllocatePagesFromEmptyChunksOnSameNode(&pGMM->Shared, pGVM, uidSelf, iPage, cPages, paPages); /* If there is a lof of free pages spread around, try not waste system memory on more chunks. (Should trigger defragmentation.) */ if ( !fTriedOnSameAlready && gmmR0ShouldAllocatePagesInOtherChunksBecauseOfLotsFree(pGMM)) { iPage = gmmR0AllocatePagesFromSameNode(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages); if (iPage < cPages) iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, uidSelf, iPage, cPages, paPages); } /* * Ok, try allocate new chunks. */ if (iPage < cPages) { do rc = gmmR0AllocateChunkNew(pGMM, pGVM, &pGMM->PrivateX, cPages, paPages, &iPage); while (iPage < cPages && RT_SUCCESS(rc)); #if 0 /* We cannot mix chunks with different UIDs. */ /* If the host is out of memory, take whatever we can get. */ if ( (rc == VERR_NO_MEMORY || rc == VERR_NO_PHYS_MEMORY) && pGMM->PrivateX.cFreePages + pGMM->Shared.cFreePages >= cPages - iPage) { iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->PrivateX, pGVM, iPage, cPages, paPages); if (iPage < cPages) iPage = gmmR0AllocatePagesIndiscriminately(&pGMM->Shared, pGVM, iPage, cPages, paPages); AssertRelease(iPage == cPages); rc = VINF_SUCCESS; } #endif } } } /* * Clean up on failure. Since this is bound to be a low-memory condition * we will give back any empty chunks that might be hanging around. */ if (RT_SUCCESS(rc)) { /* likely */ } else { /* Update the statistics. */ pGVM->gmm.s.Stats.cPrivatePages -= cPages; pGMM->cAllocatedPages -= cPages - iPage; switch (enmAccount) { case GMMACCOUNT_BASE: pGVM->gmm.s.Stats.Allocated.cBasePages -= cPages; break; case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages -= cPages; break; case GMMACCOUNT_FIXED: pGVM->gmm.s.Stats.Allocated.cFixedPages -= cPages; break; default: AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE); } /* Release the pages. */ while (iPage-- > 0) { uint32_t idPage = paPages[iPage].idPage; PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage); if (RT_LIKELY(pPage)) { Assert(GMM_PAGE_IS_PRIVATE(pPage)); Assert(pPage->Private.hGVM == pGVM->hSelf); gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage); } else AssertMsgFailed(("idPage=%#x\n", idPage)); paPages[iPage].idPage = NIL_GMM_PAGEID; paPages[iPage].idSharedPage = NIL_GMM_PAGEID; paPages[iPage].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS; paPages[iPage].fZeroed = false; } /* Free empty chunks. */ /** @todo */ /* return the fail status on failure */ return rc; } return VINF_SUCCESS; } /** * Updates the previous allocations and allocates more pages. * * The handy pages are always taken from the 'base' memory account. * The allocated pages are not cleared and will contains random garbage. * * @returns VBox status code: * @retval VINF_SUCCESS on success. * @retval VERR_NOT_OWNER if the caller is not an EMT. * @retval VERR_GMM_PAGE_NOT_FOUND if one of the pages to update wasn't found. * @retval VERR_GMM_PAGE_NOT_PRIVATE if one of the pages to update wasn't a * private page. * @retval VERR_GMM_PAGE_NOT_SHARED if one of the pages to update wasn't a * shared page. * @retval VERR_GMM_NOT_PAGE_OWNER if one of the pages to be updated wasn't * owned by the VM. * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages. * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit, * that is we're trying to allocate more than we've reserved. * * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param cPagesToUpdate The number of pages to update (starting from the head). * @param cPagesToAlloc The number of pages to allocate (starting from the head). * @param paPages The array of page descriptors. * See GMMPAGEDESC for details on what is expected on input. * @thread EMT(idCpu) */ GMMR0DECL(int) GMMR0AllocateHandyPages(PGVM pGVM, VMCPUID idCpu, uint32_t cPagesToUpdate, uint32_t cPagesToAlloc, PGMMPAGEDESC paPages) { LogFlow(("GMMR0AllocateHandyPages: pGVM=%p cPagesToUpdate=%#x cPagesToAlloc=%#x paPages=%p\n", pGVM, cPagesToUpdate, cPagesToAlloc, paPages)); /* * Validate & get basics. * (This is a relatively busy path, so make predictions where possible.) */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; AssertPtrReturn(paPages, VERR_INVALID_PARAMETER); AssertMsgReturn( (cPagesToUpdate && cPagesToUpdate < 1024) || (cPagesToAlloc && cPagesToAlloc < 1024), ("cPagesToUpdate=%#x cPagesToAlloc=%#x\n", cPagesToUpdate, cPagesToAlloc), VERR_INVALID_PARAMETER); unsigned iPage = 0; for (; iPage < cPagesToUpdate; iPage++) { AssertMsgReturn( ( paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST && !(paPages[iPage].HCPhysGCPhys & GUEST_PAGE_OFFSET_MASK)) || paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS || paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE, ("#%#x: %RHp\n", iPage, paPages[iPage].HCPhysGCPhys), VERR_INVALID_PARAMETER); /* ignore fZeroed here */ AssertMsgReturn( paPages[iPage].idPage <= GMM_PAGEID_LAST /*|| paPages[iPage].idPage == NIL_GMM_PAGEID*/, ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER); AssertMsgReturn( paPages[iPage].idSharedPage == NIL_GMM_PAGEID || paPages[iPage].idSharedPage <= GMM_PAGEID_LAST, ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER); } for (; iPage < cPagesToAlloc; iPage++) { AssertMsgReturn(paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS, ("#%#x: %RHp\n", iPage, paPages[iPage].HCPhysGCPhys), VERR_INVALID_PARAMETER); AssertMsgReturn(paPages[iPage].fZeroed == false, ("#%#x: %#x\n", iPage, paPages[iPage].fZeroed), VERR_INVALID_PARAMETER); AssertMsgReturn(paPages[iPage].idPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER); AssertMsgReturn(paPages[iPage].idSharedPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER); } /* * Take the semaphore */ VMMR0EMTBLOCKCTX Ctx; PGVMCPU pGVCpu = &pGVM->aCpus[idCpu]; rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, "GMMR0AllocateHandyPages", pGMM, &Ctx); AssertRCReturn(rc, rc); rc = gmmR0MutexAcquire(pGMM); if ( RT_SUCCESS(rc) && GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { /* No allocations before the initial reservation has been made! */ if (RT_LIKELY( pGVM->gmm.s.Stats.Reserved.cBasePages && pGVM->gmm.s.Stats.Reserved.cFixedPages && pGVM->gmm.s.Stats.Reserved.cShadowPages)) { /* * Perform the updates. * Stop on the first error. */ for (iPage = 0; iPage < cPagesToUpdate; iPage++) { if (paPages[iPage].idPage != NIL_GMM_PAGEID) { PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idPage); if (RT_LIKELY(pPage)) { if (RT_LIKELY(GMM_PAGE_IS_PRIVATE(pPage))) { if (RT_LIKELY(pPage->Private.hGVM == pGVM->hSelf)) { AssertCompile(NIL_RTHCPHYS > GMM_GCPHYS_LAST && GMM_GCPHYS_UNSHAREABLE > GMM_GCPHYS_LAST); if (RT_LIKELY(paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST)) pPage->Private.pfn = paPages[iPage].HCPhysGCPhys >> GUEST_PAGE_SHIFT; else if (paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE) pPage->Private.pfn = GMM_PAGE_PFN_UNSHAREABLE; /* else: NIL_RTHCPHYS nothing */ paPages[iPage].idPage = NIL_GMM_PAGEID; paPages[iPage].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS; paPages[iPage].fZeroed = false; } else { Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not owner! hGVM=%#x hSelf=%#x\n", iPage, paPages[iPage].idPage, pPage->Private.hGVM, pGVM->hSelf)); rc = VERR_GMM_NOT_PAGE_OWNER; break; } } else { Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not private! %.*Rhxs (type %d)\n", iPage, paPages[iPage].idPage, sizeof(*pPage), pPage, pPage->Common.u2State)); rc = VERR_GMM_PAGE_NOT_PRIVATE; break; } } else { Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not found! (private)\n", iPage, paPages[iPage].idPage)); rc = VERR_GMM_PAGE_NOT_FOUND; break; } } if (paPages[iPage].idSharedPage == NIL_GMM_PAGEID) { /* likely */ } else { PGMMPAGE pPage = gmmR0GetPage(pGMM, paPages[iPage].idSharedPage); if (RT_LIKELY(pPage)) { if (RT_LIKELY(GMM_PAGE_IS_SHARED(pPage))) { AssertCompile(NIL_RTHCPHYS > GMM_GCPHYS_LAST && GMM_GCPHYS_UNSHAREABLE > GMM_GCPHYS_LAST); Assert(pPage->Shared.cRefs); Assert(pGVM->gmm.s.Stats.cSharedPages); Assert(pGVM->gmm.s.Stats.Allocated.cBasePages); Log(("GMMR0AllocateHandyPages: free shared page %x cRefs=%d\n", paPages[iPage].idSharedPage, pPage->Shared.cRefs)); pGVM->gmm.s.Stats.cSharedPages--; pGVM->gmm.s.Stats.Allocated.cBasePages--; if (!--pPage->Shared.cRefs) gmmR0FreeSharedPage(pGMM, pGVM, paPages[iPage].idSharedPage, pPage); else { Assert(pGMM->cDuplicatePages); pGMM->cDuplicatePages--; } paPages[iPage].idSharedPage = NIL_GMM_PAGEID; } else { Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not shared!\n", iPage, paPages[iPage].idSharedPage)); rc = VERR_GMM_PAGE_NOT_SHARED; break; } } else { Log(("GMMR0AllocateHandyPages: #%#x/%#x: Not found! (shared)\n", iPage, paPages[iPage].idSharedPage)); rc = VERR_GMM_PAGE_NOT_FOUND; break; } } } /* for each page to update */ if (RT_SUCCESS(rc) && cPagesToAlloc > 0) { #ifdef VBOX_STRICT for (iPage = 0; iPage < cPagesToAlloc; iPage++) { Assert(paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS); Assert(paPages[iPage].fZeroed == false); Assert(paPages[iPage].idPage == NIL_GMM_PAGEID); Assert(paPages[iPage].idSharedPage == NIL_GMM_PAGEID); } #endif /* * Join paths with GMMR0AllocatePages for the allocation. * Note! gmmR0AllocateMoreChunks may leave the protection of the mutex! */ rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPagesToAlloc, paPages, GMMACCOUNT_BASE); } } else rc = VERR_WRONG_ORDER; GMM_CHECK_SANITY_UPON_LEAVING(pGMM); gmmR0MutexRelease(pGMM); } else if (RT_SUCCESS(rc)) { gmmR0MutexRelease(pGMM); rc = VERR_GMM_IS_NOT_SANE; } VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx); LogFlow(("GMMR0AllocateHandyPages: returns %Rrc\n", rc)); return rc; } /** * Allocate one or more pages. * * This is typically used for ROMs and MMIO2 (VRAM) during VM creation. * The allocated pages are not cleared and will contain random garbage. * * @returns VBox status code: * @retval VINF_SUCCESS on success. * @retval VERR_NOT_OWNER if the caller is not an EMT. * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages. * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit, * that is we're trying to allocate more than we've reserved. * * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param cPages The number of pages to allocate. * @param paPages Pointer to the page descriptors. * See GMMPAGEDESC for details on what is expected on * input. * @param enmAccount The account to charge. * * @thread EMT. */ GMMR0DECL(int) GMMR0AllocatePages(PGVM pGVM, VMCPUID idCpu, uint32_t cPages, PGMMPAGEDESC paPages, GMMACCOUNT enmAccount) { LogFlow(("GMMR0AllocatePages: pGVM=%p cPages=%#x paPages=%p enmAccount=%d\n", pGVM, cPages, paPages, enmAccount)); /* * Validate, get basics and take the semaphore. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; AssertPtrReturn(paPages, VERR_INVALID_PARAMETER); AssertMsgReturn(enmAccount > GMMACCOUNT_INVALID && enmAccount < GMMACCOUNT_END, ("%d\n", enmAccount), VERR_INVALID_PARAMETER); AssertMsgReturn(cPages > 0 && cPages < RT_BIT(32 - GUEST_PAGE_SHIFT), ("%#x\n", cPages), VERR_INVALID_PARAMETER); for (unsigned iPage = 0; iPage < cPages; iPage++) { AssertMsgReturn( paPages[iPage].HCPhysGCPhys == NIL_GMMPAGEDESC_PHYS || paPages[iPage].HCPhysGCPhys == GMM_GCPHYS_UNSHAREABLE || ( enmAccount == GMMACCOUNT_BASE && paPages[iPage].HCPhysGCPhys <= GMM_GCPHYS_LAST && !(paPages[iPage].HCPhysGCPhys & GUEST_PAGE_OFFSET_MASK)), ("#%#x: %RHp enmAccount=%d\n", iPage, paPages[iPage].HCPhysGCPhys, enmAccount), VERR_INVALID_PARAMETER); AssertMsgReturn(paPages[iPage].fZeroed == false, ("#%#x: %#x\n", iPage, paPages[iPage].fZeroed), VERR_INVALID_PARAMETER); AssertMsgReturn(paPages[iPage].idPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER); AssertMsgReturn(paPages[iPage].idSharedPage == NIL_GMM_PAGEID, ("#%#x: %#x\n", iPage, paPages[iPage].idSharedPage), VERR_INVALID_PARAMETER); } /* * Grab the giant mutex and get working. */ gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { /* No allocations before the initial reservation has been made! */ if (RT_LIKELY( pGVM->gmm.s.Stats.Reserved.cBasePages && pGVM->gmm.s.Stats.Reserved.cFixedPages && pGVM->gmm.s.Stats.Reserved.cShadowPages)) rc = gmmR0AllocatePagesNew(pGMM, pGVM, cPages, paPages, enmAccount); else rc = VERR_WRONG_ORDER; GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); LogFlow(("GMMR0AllocatePages: returns %Rrc\n", rc)); return rc; } /** * VMMR0 request wrapper for GMMR0AllocatePages. * * @returns see GMMR0AllocatePages. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0AllocatePagesReq(PGVM pGVM, VMCPUID idCpu, PGMMALLOCATEPAGESREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF(GMMALLOCATEPAGESREQ, aPages[0]), ("%#x < %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF(GMMALLOCATEPAGESREQ, aPages[0])), VERR_INVALID_PARAMETER); AssertMsgReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMALLOCATEPAGESREQ, aPages[pReq->cPages]), ("%#x != %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF_DYN(GMMALLOCATEPAGESREQ, aPages[pReq->cPages])), VERR_INVALID_PARAMETER); return GMMR0AllocatePages(pGVM, idCpu, pReq->cPages, &pReq->aPages[0], pReq->enmAccount); } /** * Allocate a large page to represent guest RAM * * The allocated pages are zeroed upon return. * * @returns VBox status code: * @retval VINF_SUCCESS on success. * @retval VERR_NOT_OWNER if the caller is not an EMT. * @retval VERR_GMM_HIT_GLOBAL_LIMIT if we've exhausted the available pages. * @retval VERR_GMM_HIT_VM_ACCOUNT_LIMIT if we've hit the VM account limit, * that is we're trying to allocate more than we've reserved. * @retval VERR_TRY_AGAIN if the host is temporarily out of large pages. * @returns see GMMR0AllocatePages. * * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param cbPage Large page size. * @param pIdPage Where to return the GMM page ID of the page. * @param pHCPhys Where to return the host physical address of the page. */ GMMR0DECL(int) GMMR0AllocateLargePage(PGVM pGVM, VMCPUID idCpu, uint32_t cbPage, uint32_t *pIdPage, RTHCPHYS *pHCPhys) { LogFlow(("GMMR0AllocateLargePage: pGVM=%p cbPage=%x\n", pGVM, cbPage)); AssertPtrReturn(pIdPage, VERR_INVALID_PARAMETER); *pIdPage = NIL_GMM_PAGEID; AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER); *pHCPhys = NIL_RTHCPHYS; AssertReturn(cbPage == GMM_CHUNK_SIZE, VERR_INVALID_PARAMETER); /* * Validate GVM + idCpu, get basics and take the semaphore. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); AssertRCReturn(rc, rc); VMMR0EMTBLOCKCTX Ctx; PGVMCPU pGVCpu = &pGVM->aCpus[idCpu]; rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, "GMMR0AllocateLargePage", pGMM, &Ctx); AssertRCReturn(rc, rc); rc = gmmR0MutexAcquire(pGMM); if (RT_SUCCESS(rc)) { if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { /* * Check the quota. */ /** @todo r=bird: Quota checking could be done w/o the giant mutex but using * a VM specific mutex... */ if (RT_LIKELY( pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + GMM_CHUNK_NUM_PAGES <= pGVM->gmm.s.Stats.Reserved.cBasePages)) { /* * Allocate a new large page chunk. * * Note! We leave the giant GMM lock temporarily as the allocation might * take a long time. gmmR0RegisterChunk will retake it (ugly). */ AssertCompile(GMM_CHUNK_SIZE == _2M); gmmR0MutexRelease(pGMM); RTR0MEMOBJ hMemObj; rc = RTR0MemObjAllocLarge(&hMemObj, GMM_CHUNK_SIZE, GMM_CHUNK_SIZE, RTMEMOBJ_ALLOC_LARGE_F_FAST); if (RT_SUCCESS(rc)) { *pHCPhys = RTR0MemObjGetPagePhysAddr(hMemObj, 0); /* * Register the chunk as fully allocated. * Note! As mentioned above, this will return owning the mutex on success. */ PGMMCHUNK pChunk = NULL; PGMMCHUNKFREESET const pSet = pGMM->fBoundMemoryMode ? &pGVM->gmm.s.Private : &pGMM->PrivateX; rc = gmmR0RegisterChunk(pGMM, pSet, hMemObj, pGVM->hSelf, pGVM->pSession, GMM_CHUNK_FLAGS_LARGE_PAGE, 0 /*cPages*/, NULL /*paPages*/, NULL /*piPage*/, &pChunk); if (RT_SUCCESS(rc)) { /* * The gmmR0RegisterChunk call already marked all pages allocated, * so we just have to fill in the return values and update stats now. */ *pIdPage = pChunk->Core.Key << GMM_CHUNKID_SHIFT; /* Update accounting. */ pGVM->gmm.s.Stats.Allocated.cBasePages += GMM_CHUNK_NUM_PAGES; pGVM->gmm.s.Stats.cPrivatePages += GMM_CHUNK_NUM_PAGES; pGMM->cAllocatedPages += GMM_CHUNK_NUM_PAGES; gmmR0LinkChunk(pChunk, pSet); gmmR0MutexRelease(pGMM); VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx); LogFlow(("GMMR0AllocateLargePage: returns VINF_SUCCESS\n")); return VINF_SUCCESS; } /* * Bail out. */ RTR0MemObjFree(hMemObj, true /* fFreeMappings */); *pHCPhys = NIL_RTHCPHYS; } /** @todo r=bird: Turn VERR_NO_MEMORY etc into VERR_TRY_AGAIN? Docs say we * return it, but I am sure IPRT doesn't... */ } else { Log(("GMMR0AllocateLargePage: Reserved=%#llx Allocated+Requested=%#llx+%#x!\n", pGVM->gmm.s.Stats.Reserved.cBasePages, pGVM->gmm.s.Stats.Allocated.cBasePages, GMM_CHUNK_NUM_PAGES)); gmmR0MutexRelease(pGMM); rc = VERR_GMM_HIT_VM_ACCOUNT_LIMIT; } } else { gmmR0MutexRelease(pGMM); rc = VERR_GMM_IS_NOT_SANE; } } VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx); LogFlow(("GMMR0AllocateLargePage: returns %Rrc\n", rc)); return rc; } /** * Free a large page. * * @returns VBox status code: * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param idPage The large page id. */ GMMR0DECL(int) GMMR0FreeLargePage(PGVM pGVM, VMCPUID idCpu, uint32_t idPage) { LogFlow(("GMMR0FreeLargePage: pGVM=%p idPage=%x\n", pGVM, idPage)); /* * Validate, get basics and take the semaphore. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { const unsigned cPages = GMM_CHUNK_NUM_PAGES; if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages < cPages)) { Log(("GMMR0FreeLargePage: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cBasePages, cPages)); gmmR0MutexRelease(pGMM); return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH; } PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage); if (RT_LIKELY( pPage && GMM_PAGE_IS_PRIVATE(pPage))) { PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT); Assert(pChunk); Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES); Assert(pChunk->cPrivate > 0); /* Release the memory immediately. */ gmmR0FreeChunk(pGMM, NULL, pChunk, false /*fRelaxedSem*/); /** @todo this can be relaxed too! */ /* Update accounting. */ pGVM->gmm.s.Stats.Allocated.cBasePages -= cPages; pGVM->gmm.s.Stats.cPrivatePages -= cPages; pGMM->cAllocatedPages -= cPages; } else rc = VERR_GMM_PAGE_NOT_FOUND; } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); LogFlow(("GMMR0FreeLargePage: returns %Rrc\n", rc)); return rc; } /** * VMMR0 request wrapper for GMMR0FreeLargePage. * * @returns see GMMR0FreeLargePage. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0FreeLargePageReq(PGVM pGVM, VMCPUID idCpu, PGMMFREELARGEPAGEREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMFREEPAGESREQ), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(GMMFREEPAGESREQ)), VERR_INVALID_PARAMETER); return GMMR0FreeLargePage(pGVM, idCpu, pReq->idPage); } /** * @callback_method_impl{FNGVMMR0ENUMCALLBACK, * Used by gmmR0FreeChunkFlushPerVmTlbs().} */ static DECLCALLBACK(int) gmmR0InvalidatePerVmChunkTlbCallback(PGVM pGVM, void *pvUser) { RT_NOREF(pvUser); if (pGVM->gmm.s.hChunkTlbSpinLock != NIL_RTSPINLOCK) { RTSpinlockAcquire(pGVM->gmm.s.hChunkTlbSpinLock); uintptr_t i = RT_ELEMENTS(pGVM->gmm.s.aChunkTlbEntries); while (i-- > 0) { pGVM->gmm.s.aChunkTlbEntries[i].idGeneration = UINT64_MAX; pGVM->gmm.s.aChunkTlbEntries[i].pChunk = NULL; } RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock); } return VINF_SUCCESS; } /** * Called by gmmR0FreeChunk when we reach the threshold for wrapping around the * free generation ID value. * * This is done at 2^62 - 1, which allows us to drop all locks and as it will * take a while before 12 exa (2 305 843 009 213 693 952) calls to * gmmR0FreeChunk can be made and causes a real wrap-around. We do two * invalidation passes and resets the generation ID between then. This will * make sure there are no false positives. * * @param pGMM Pointer to the GMM instance. */ static void gmmR0FreeChunkFlushPerVmTlbs(PGMM pGMM) { /* * First invalidation pass. */ int rc = GVMMR0EnumVMs(gmmR0InvalidatePerVmChunkTlbCallback, NULL); AssertRCSuccess(rc); /* * Reset the generation number. */ RTSpinlockAcquire(pGMM->hSpinLockTree); ASMAtomicWriteU64(&pGMM->idFreeGeneration, 1); RTSpinlockRelease(pGMM->hSpinLockTree); /* * Second invalidation pass. */ rc = GVMMR0EnumVMs(gmmR0InvalidatePerVmChunkTlbCallback, NULL); AssertRCSuccess(rc); } /** * Frees a chunk, giving it back to the host OS. * * @param pGMM Pointer to the GMM instance. * @param pGVM This is set when called from GMMR0CleanupVM so we can * unmap and free the chunk in one go. * @param pChunk The chunk to free. * @param fRelaxedSem Whether we can release the semaphore while doing the * freeing (@c true) or not. */ static bool gmmR0FreeChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem) { Assert(pChunk->Core.Key != NIL_GMM_CHUNKID); GMMR0CHUNKMTXSTATE MtxState; gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT); /* * Cleanup hack! Unmap the chunk from the callers address space. * This shouldn't happen, so screw lock contention... */ if (pChunk->cMappingsX && pGVM) gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk); /* * If there are current mappings of the chunk, then request the * VMs to unmap them. Reposition the chunk in the free list so * it won't be a likely candidate for allocations. */ if (pChunk->cMappingsX) { /** @todo R0 -> VM request */ /* The chunk can be mapped by more than one VM if fBoundMemoryMode is false! */ Log(("gmmR0FreeChunk: chunk still has %d mappings; don't free!\n", pChunk->cMappingsX)); gmmR0ChunkMutexRelease(&MtxState, pChunk); return false; } /* * Save and trash the handle. */ RTR0MEMOBJ const hMemObj = pChunk->hMemObj; pChunk->hMemObj = NIL_RTR0MEMOBJ; /* * Unlink it from everywhere. */ gmmR0UnlinkChunk(pChunk); RTSpinlockAcquire(pGMM->hSpinLockTree); RTListNodeRemove(&pChunk->ListNode); PAVLU32NODECORE pCore = RTAvlU32Remove(&pGMM->pChunks, pChunk->Core.Key); Assert(pCore == &pChunk->Core); NOREF(pCore); PGMMCHUNKTLBE pTlbe = &pGMM->ChunkTLB.aEntries[GMM_CHUNKTLB_IDX(pChunk->Core.Key)]; if (pTlbe->pChunk == pChunk) { pTlbe->idChunk = NIL_GMM_CHUNKID; pTlbe->pChunk = NULL; } Assert(pGMM->cChunks > 0); pGMM->cChunks--; uint64_t const idFreeGeneration = ASMAtomicIncU64(&pGMM->idFreeGeneration); RTSpinlockRelease(pGMM->hSpinLockTree); pGMM->cFreedChunks++; /* Drop the lock. */ gmmR0ChunkMutexRelease(&MtxState, NULL); if (fRelaxedSem) gmmR0MutexRelease(pGMM); /* * Flush per VM chunk TLBs if we're getting remotely close to a generation wraparound. */ if (idFreeGeneration == UINT64_MAX / 4) gmmR0FreeChunkFlushPerVmTlbs(pGMM); /* * Free the Chunk ID and all memory associated with the chunk. */ gmmR0FreeChunkId(pGMM, pChunk->Core.Key); pChunk->Core.Key = NIL_GMM_CHUNKID; RTMemFree(pChunk->paMappingsX); pChunk->paMappingsX = NULL; RTMemFree(pChunk); #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM int rc = RTR0MemObjFree(hMemObj, true /* fFreeMappings */); #else int rc = RTR0MemObjFree(hMemObj, false /* fFreeMappings */); #endif AssertLogRelRC(rc); if (fRelaxedSem) gmmR0MutexAcquire(pGMM); return fRelaxedSem; } /** * Free page worker. * * The caller does all the statistic decrementing, we do all the incrementing. * * @param pGMM Pointer to the GMM instance data. * @param pGVM Pointer to the GVM instance. * @param pChunk Pointer to the chunk this page belongs to. * @param idPage The Page ID. * @param pPage Pointer to the page. */ static void gmmR0FreePageWorker(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, uint32_t idPage, PGMMPAGE pPage) { Log3(("F pPage=%p iPage=%#x/%#x u2State=%d iFreeHead=%#x\n", pPage, pPage - &pChunk->aPages[0], idPage, pPage->Common.u2State, pChunk->iFreeHead)); NOREF(idPage); /* * Put the page on the free list. */ pPage->u = 0; pPage->Free.u2State = GMM_PAGE_STATE_FREE; pPage->Free.fZeroed = false; Assert(pChunk->iFreeHead < RT_ELEMENTS(pChunk->aPages) || pChunk->iFreeHead == UINT16_MAX); pPage->Free.iNext = pChunk->iFreeHead; pChunk->iFreeHead = pPage - &pChunk->aPages[0]; /* * Update statistics (the cShared/cPrivate stats are up to date already), * and relink the chunk if necessary. */ unsigned const cFree = pChunk->cFree; if ( !cFree || gmmR0SelectFreeSetList(cFree) != gmmR0SelectFreeSetList(cFree + 1)) { gmmR0UnlinkChunk(pChunk); pChunk->cFree++; gmmR0SelectSetAndLinkChunk(pGMM, pGVM, pChunk); } else { pChunk->cFree = cFree + 1; pChunk->pSet->cFreePages++; } /* * If the chunk becomes empty, consider giving memory back to the host OS. * * The current strategy is to try give it back if there are other chunks * in this free list, meaning if there are at least 240 free pages in this * category. Note that since there are probably mappings of the chunk, * it won't be freed up instantly, which probably screws up this logic * a bit... */ /** @todo Do this on the way out. */ if (RT_LIKELY( pChunk->cFree != GMM_CHUNK_NUM_PAGES || pChunk->pFreeNext == NULL || pChunk->pFreePrev == NULL /** @todo this is probably misfiring, see reset... */)) { /* likely */ } else gmmR0FreeChunk(pGMM, NULL, pChunk, false); } /** * Frees a shared page, the page is known to exist and be valid and such. * * @param pGMM Pointer to the GMM instance. * @param pGVM Pointer to the GVM instance. * @param idPage The page id. * @param pPage The page structure. */ DECLINLINE(void) gmmR0FreeSharedPage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage) { PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT); Assert(pChunk); Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES); Assert(pChunk->cShared > 0); Assert(pGMM->cSharedPages > 0); Assert(pGMM->cAllocatedPages > 0); Assert(!pPage->Shared.cRefs); pChunk->cShared--; pGMM->cAllocatedPages--; pGMM->cSharedPages--; gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage); } /** * Frees a private page, the page is known to exist and be valid and such. * * @param pGMM Pointer to the GMM instance. * @param pGVM Pointer to the GVM instance. * @param idPage The page id. * @param pPage The page structure. */ DECLINLINE(void) gmmR0FreePrivatePage(PGMM pGMM, PGVM pGVM, uint32_t idPage, PGMMPAGE pPage) { PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT); Assert(pChunk); Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES); Assert(pChunk->cPrivate > 0); Assert(pGMM->cAllocatedPages > 0); pChunk->cPrivate--; pGMM->cAllocatedPages--; gmmR0FreePageWorker(pGMM, pGVM, pChunk, idPage, pPage); } /** * Common worker for GMMR0FreePages and GMMR0BalloonedPages. * * @returns VBox status code: * @retval xxx * * @param pGMM Pointer to the GMM instance data. * @param pGVM Pointer to the VM. * @param cPages The number of pages to free. * @param paPages Pointer to the page descriptors. * @param enmAccount The account this relates to. */ static int gmmR0FreePages(PGMM pGMM, PGVM pGVM, uint32_t cPages, PGMMFREEPAGEDESC paPages, GMMACCOUNT enmAccount) { /* * Check that the request isn't impossible wrt to the account status. */ switch (enmAccount) { case GMMACCOUNT_BASE: if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages < cPages)) { Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cBasePages, cPages)); return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH; } break; case GMMACCOUNT_SHADOW: if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cShadowPages < cPages)) { Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cShadowPages, cPages)); return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH; } break; case GMMACCOUNT_FIXED: if (RT_UNLIKELY(pGVM->gmm.s.Stats.Allocated.cFixedPages < cPages)) { Log(("gmmR0FreePages: allocated=%#llx cPages=%#x!\n", pGVM->gmm.s.Stats.Allocated.cFixedPages, cPages)); return VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH; } break; default: AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE); } /* * Walk the descriptors and free the pages. * * Statistics (except the account) are being updated as we go along, * unlike the alloc code. Also, stop on the first error. */ int rc = VINF_SUCCESS; uint32_t iPage; for (iPage = 0; iPage < cPages; iPage++) { uint32_t idPage = paPages[iPage].idPage; PGMMPAGE pPage = gmmR0GetPage(pGMM, idPage); if (RT_LIKELY(pPage)) { if (RT_LIKELY(GMM_PAGE_IS_PRIVATE(pPage))) { if (RT_LIKELY(pPage->Private.hGVM == pGVM->hSelf)) { Assert(pGVM->gmm.s.Stats.cPrivatePages); pGVM->gmm.s.Stats.cPrivatePages--; gmmR0FreePrivatePage(pGMM, pGVM, idPage, pPage); } else { Log(("gmmR0AllocatePages: #%#x/%#x: not owner! hGVM=%#x hSelf=%#x\n", iPage, idPage, pPage->Private.hGVM, pGVM->hSelf)); rc = VERR_GMM_NOT_PAGE_OWNER; break; } } else if (RT_LIKELY(GMM_PAGE_IS_SHARED(pPage))) { Assert(pGVM->gmm.s.Stats.cSharedPages); Assert(pPage->Shared.cRefs); #if defined(VBOX_WITH_PAGE_SHARING) && defined(VBOX_STRICT) if (pPage->Shared.u14Checksum) { uint32_t uChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage); uChecksum &= UINT32_C(0x00003fff); AssertMsg(!uChecksum || uChecksum == pPage->Shared.u14Checksum, ("%#x vs %#x - idPage=%#x\n", uChecksum, pPage->Shared.u14Checksum, idPage)); } #endif pGVM->gmm.s.Stats.cSharedPages--; if (!--pPage->Shared.cRefs) gmmR0FreeSharedPage(pGMM, pGVM, idPage, pPage); else { Assert(pGMM->cDuplicatePages); pGMM->cDuplicatePages--; } } else { Log(("gmmR0AllocatePages: #%#x/%#x: already free!\n", iPage, idPage)); rc = VERR_GMM_PAGE_ALREADY_FREE; break; } } else { Log(("gmmR0AllocatePages: #%#x/%#x: not found!\n", iPage, idPage)); rc = VERR_GMM_PAGE_NOT_FOUND; break; } paPages[iPage].idPage = NIL_GMM_PAGEID; } /* * Update the account. */ switch (enmAccount) { case GMMACCOUNT_BASE: pGVM->gmm.s.Stats.Allocated.cBasePages -= iPage; break; case GMMACCOUNT_SHADOW: pGVM->gmm.s.Stats.Allocated.cShadowPages -= iPage; break; case GMMACCOUNT_FIXED: pGVM->gmm.s.Stats.Allocated.cFixedPages -= iPage; break; default: AssertMsgFailedReturn(("enmAccount=%d\n", enmAccount), VERR_IPE_NOT_REACHED_DEFAULT_CASE); } /* * Any threshold stuff to be done here? */ return rc; } /** * Free one or more pages. * * This is typically used at reset time or power off. * * @returns VBox status code: * @retval xxx * * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param cPages The number of pages to allocate. * @param paPages Pointer to the page descriptors containing the page IDs * for each page. * @param enmAccount The account this relates to. * @thread EMT. */ GMMR0DECL(int) GMMR0FreePages(PGVM pGVM, VMCPUID idCpu, uint32_t cPages, PGMMFREEPAGEDESC paPages, GMMACCOUNT enmAccount) { LogFlow(("GMMR0FreePages: pGVM=%p cPages=%#x paPages=%p enmAccount=%d\n", pGVM, cPages, paPages, enmAccount)); /* * Validate input and get the basics. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; AssertPtrReturn(paPages, VERR_INVALID_PARAMETER); AssertMsgReturn(enmAccount > GMMACCOUNT_INVALID && enmAccount < GMMACCOUNT_END, ("%d\n", enmAccount), VERR_INVALID_PARAMETER); AssertMsgReturn(cPages > 0 && cPages < RT_BIT(32 - GUEST_PAGE_SHIFT), ("%#x\n", cPages), VERR_INVALID_PARAMETER); for (unsigned iPage = 0; iPage < cPages; iPage++) AssertMsgReturn( paPages[iPage].idPage <= GMM_PAGEID_LAST /*|| paPages[iPage].idPage == NIL_GMM_PAGEID*/, ("#%#x: %#x\n", iPage, paPages[iPage].idPage), VERR_INVALID_PARAMETER); /* * Take the semaphore and call the worker function. */ gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { rc = gmmR0FreePages(pGMM, pGVM, cPages, paPages, enmAccount); GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); LogFlow(("GMMR0FreePages: returns %Rrc\n", rc)); return rc; } /** * VMMR0 request wrapper for GMMR0FreePages. * * @returns see GMMR0FreePages. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0FreePagesReq(PGVM pGVM, VMCPUID idCpu, PGMMFREEPAGESREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF(GMMFREEPAGESREQ, aPages[0]), ("%#x < %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF(GMMFREEPAGESREQ, aPages[0])), VERR_INVALID_PARAMETER); AssertMsgReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[pReq->cPages]), ("%#x != %#x\n", pReq->Hdr.cbReq, RT_UOFFSETOF_DYN(GMMFREEPAGESREQ, aPages[pReq->cPages])), VERR_INVALID_PARAMETER); return GMMR0FreePages(pGVM, idCpu, pReq->cPages, &pReq->aPages[0], pReq->enmAccount); } /** * Report back on a memory ballooning request. * * The request may or may not have been initiated by the GMM. If it was initiated * by the GMM it is important that this function is called even if no pages were * ballooned. * * @returns VBox status code: * @retval VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH * @retval VERR_GMM_ATTEMPT_TO_DEFLATE_TOO_MUCH * @retval VERR_GMM_OVERCOMMITTED_TRY_AGAIN_IN_A_BIT - reset condition * indicating that we won't necessarily have sufficient RAM to boot * the VM again and that it should pause until this changes (we'll try * balloon some other VM). (For standard deflate we have little choice * but to hope the VM won't use the memory that was returned to it.) * * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param enmAction Inflate/deflate/reset. * @param cBalloonedPages The number of pages that was ballooned. * * @thread EMT(idCpu) */ GMMR0DECL(int) GMMR0BalloonedPages(PGVM pGVM, VMCPUID idCpu, GMMBALLOONACTION enmAction, uint32_t cBalloonedPages) { LogFlow(("GMMR0BalloonedPages: pGVM=%p enmAction=%d cBalloonedPages=%#x\n", pGVM, enmAction, cBalloonedPages)); AssertMsgReturn(cBalloonedPages < RT_BIT(32 - GUEST_PAGE_SHIFT), ("%#x\n", cBalloonedPages), VERR_INVALID_PARAMETER); /* * Validate input and get the basics. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; /* * Take the semaphore and do some more validations. */ gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { switch (enmAction) { case GMMBALLOONACTION_INFLATE: { if (RT_LIKELY(pGVM->gmm.s.Stats.Allocated.cBasePages + pGVM->gmm.s.Stats.cBalloonedPages + cBalloonedPages <= pGVM->gmm.s.Stats.Reserved.cBasePages)) { /* * Record the ballooned memory. */ pGMM->cBalloonedPages += cBalloonedPages; if (pGVM->gmm.s.Stats.cReqBalloonedPages) { /* Codepath never taken. Might be interesting in the future to request ballooned memory from guests in low memory conditions.. */ AssertFailed(); pGVM->gmm.s.Stats.cBalloonedPages += cBalloonedPages; pGVM->gmm.s.Stats.cReqActuallyBalloonedPages += cBalloonedPages; Log(("GMMR0BalloonedPages: +%#x - Global=%#llx / VM: Total=%#llx Req=%#llx Actual=%#llx (pending)\n", cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages, pGVM->gmm.s.Stats.cReqBalloonedPages, pGVM->gmm.s.Stats.cReqActuallyBalloonedPages)); } else { pGVM->gmm.s.Stats.cBalloonedPages += cBalloonedPages; Log(("GMMR0BalloonedPages: +%#x - Global=%#llx / VM: Total=%#llx (user)\n", cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages)); } } else { Log(("GMMR0BalloonedPages: cBasePages=%#llx Total=%#llx cBalloonedPages=%#llx Reserved=%#llx\n", pGVM->gmm.s.Stats.Allocated.cBasePages, pGVM->gmm.s.Stats.cBalloonedPages, cBalloonedPages, pGVM->gmm.s.Stats.Reserved.cBasePages)); rc = VERR_GMM_ATTEMPT_TO_FREE_TOO_MUCH; } break; } case GMMBALLOONACTION_DEFLATE: { /* Deflate. */ if (pGVM->gmm.s.Stats.cBalloonedPages >= cBalloonedPages) { /* * Record the ballooned memory. */ Assert(pGMM->cBalloonedPages >= cBalloonedPages); pGMM->cBalloonedPages -= cBalloonedPages; pGVM->gmm.s.Stats.cBalloonedPages -= cBalloonedPages; if (pGVM->gmm.s.Stats.cReqDeflatePages) { AssertFailed(); /* This is path is for later. */ Log(("GMMR0BalloonedPages: -%#x - Global=%#llx / VM: Total=%#llx Req=%#llx\n", cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages, pGVM->gmm.s.Stats.cReqDeflatePages)); /* * Anything we need to do here now when the request has been completed? */ pGVM->gmm.s.Stats.cReqDeflatePages = 0; } else Log(("GMMR0BalloonedPages: -%#x - Global=%#llx / VM: Total=%#llx (user)\n", cBalloonedPages, pGMM->cBalloonedPages, pGVM->gmm.s.Stats.cBalloonedPages)); } else { Log(("GMMR0BalloonedPages: Total=%#llx cBalloonedPages=%#llx\n", pGVM->gmm.s.Stats.cBalloonedPages, cBalloonedPages)); rc = VERR_GMM_ATTEMPT_TO_DEFLATE_TOO_MUCH; } break; } case GMMBALLOONACTION_RESET: { /* Reset to an empty balloon. */ Assert(pGMM->cBalloonedPages >= pGVM->gmm.s.Stats.cBalloonedPages); pGMM->cBalloonedPages -= pGVM->gmm.s.Stats.cBalloonedPages; pGVM->gmm.s.Stats.cBalloonedPages = 0; break; } default: rc = VERR_INVALID_PARAMETER; break; } GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); LogFlow(("GMMR0BalloonedPages: returns %Rrc\n", rc)); return rc; } /** * VMMR0 request wrapper for GMMR0BalloonedPages. * * @returns see GMMR0BalloonedPages. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0BalloonedPagesReq(PGVM pGVM, VMCPUID idCpu, PGMMBALLOONEDPAGESREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMBALLOONEDPAGESREQ), ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMBALLOONEDPAGESREQ)), VERR_INVALID_PARAMETER); return GMMR0BalloonedPages(pGVM, idCpu, pReq->enmAction, pReq->cBalloonedPages); } /** * Return memory statistics for the hypervisor * * @returns VBox status code. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0QueryHypervisorMemoryStatsReq(PGMMMEMSTATSREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMMEMSTATSREQ), ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMMEMSTATSREQ)), VERR_INVALID_PARAMETER); /* * Validate input and get the basics. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); pReq->cAllocPages = pGMM->cAllocatedPages; pReq->cFreePages = (pGMM->cChunks << (GMM_CHUNK_SHIFT - GUEST_PAGE_SHIFT)) - pGMM->cAllocatedPages; pReq->cBalloonedPages = pGMM->cBalloonedPages; pReq->cMaxPages = pGMM->cMaxPages; pReq->cSharedPages = pGMM->cDuplicatePages; GMM_CHECK_SANITY_UPON_LEAVING(pGMM); return VINF_SUCCESS; } /** * Return memory statistics for the VM * * @returns VBox status code. * @param pGVM The global (ring-0) VM structure. * @param idCpu Cpu id. * @param pReq Pointer to the request packet. * * @thread EMT(idCpu) */ GMMR0DECL(int) GMMR0QueryMemoryStatsReq(PGVM pGVM, VMCPUID idCpu, PGMMMEMSTATSREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(GMMMEMSTATSREQ), ("%#x < %#x\n", pReq->Hdr.cbReq, sizeof(GMMMEMSTATSREQ)), VERR_INVALID_PARAMETER); /* * Validate input and get the basics. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; /* * Take the semaphore and do some more validations. */ gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { pReq->cAllocPages = pGVM->gmm.s.Stats.Allocated.cBasePages; pReq->cBalloonedPages = pGVM->gmm.s.Stats.cBalloonedPages; pReq->cMaxPages = pGVM->gmm.s.Stats.Reserved.cBasePages; pReq->cFreePages = pReq->cMaxPages - pReq->cAllocPages; } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); LogFlow(("GMMR3QueryVMMemoryStats: returns %Rrc\n", rc)); return rc; } /** * Worker for gmmR0UnmapChunk and gmmr0FreeChunk. * * Don't call this in legacy allocation mode! * * @returns VBox status code. * @param pGMM Pointer to the GMM instance data. * @param pGVM Pointer to the Global VM structure. * @param pChunk Pointer to the chunk to be unmapped. */ static int gmmR0UnmapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk) { RT_NOREF_PV(pGMM); /* * Find the mapping and try unmapping it. */ uint32_t cMappings = pChunk->cMappingsX; for (uint32_t i = 0; i < cMappings; i++) { Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ); if (pChunk->paMappingsX[i].pGVM == pGVM) { /* unmap */ int rc = RTR0MemObjFree(pChunk->paMappingsX[i].hMapObj, false /* fFreeMappings (NA) */); if (RT_SUCCESS(rc)) { /* update the record. */ cMappings--; if (i < cMappings) pChunk->paMappingsX[i] = pChunk->paMappingsX[cMappings]; pChunk->paMappingsX[cMappings].hMapObj = NIL_RTR0MEMOBJ; pChunk->paMappingsX[cMappings].pGVM = NULL; Assert(pChunk->cMappingsX - 1U == cMappings); pChunk->cMappingsX = cMappings; } return rc; } } Log(("gmmR0UnmapChunk: Chunk %#x is not mapped into pGVM=%p/%#x\n", pChunk->Core.Key, pGVM, pGVM->hSelf)); return VERR_GMM_CHUNK_NOT_MAPPED; } /** * Unmaps a chunk previously mapped into the address space of the current process. * * @returns VBox status code. * @param pGMM Pointer to the GMM instance data. * @param pGVM Pointer to the Global VM structure. * @param pChunk Pointer to the chunk to be unmapped. * @param fRelaxedSem Whether we can release the semaphore while doing the * mapping (@c true) or not. */ static int gmmR0UnmapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem) { /* * Lock the chunk and if possible leave the giant GMM lock. */ GMMR0CHUNKMTXSTATE MtxState; int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, fRelaxedSem ? GMMR0CHUNK_MTX_RETAKE_GIANT : GMMR0CHUNK_MTX_KEEP_GIANT); if (RT_SUCCESS(rc)) { rc = gmmR0UnmapChunkLocked(pGMM, pGVM, pChunk); gmmR0ChunkMutexRelease(&MtxState, pChunk); } return rc; } /** * Worker for gmmR0MapChunk. * * @returns VBox status code. * @param pGMM Pointer to the GMM instance data. * @param pGVM Pointer to the Global VM structure. * @param pChunk Pointer to the chunk to be mapped. * @param ppvR3 Where to store the ring-3 address of the mapping. * In the VERR_GMM_CHUNK_ALREADY_MAPPED case, this will be * contain the address of the existing mapping. */ static int gmmR0MapChunkLocked(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3) { RT_NOREF(pGMM); /* * Check to see if the chunk is already mapped. */ for (uint32_t i = 0; i < pChunk->cMappingsX; i++) { Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ); if (pChunk->paMappingsX[i].pGVM == pGVM) { *ppvR3 = RTR0MemObjAddressR3(pChunk->paMappingsX[i].hMapObj); Log(("gmmR0MapChunk: chunk %#x is already mapped at %p!\n", pChunk->Core.Key, *ppvR3)); #ifdef VBOX_WITH_PAGE_SHARING /* The ring-3 chunk cache can be out of sync; don't fail. */ return VINF_SUCCESS; #else return VERR_GMM_CHUNK_ALREADY_MAPPED; #endif } } /* * Do the mapping. */ RTR0MEMOBJ hMapObj; int rc = RTR0MemObjMapUser(&hMapObj, pChunk->hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS); if (RT_SUCCESS(rc)) { /* reallocate the array? assumes few users per chunk (usually one). */ unsigned iMapping = pChunk->cMappingsX; if ( iMapping <= 3 || (iMapping & 3) == 0) { unsigned cNewSize = iMapping <= 3 ? iMapping + 1 : iMapping + 4; Assert(cNewSize < 4 || RT_ALIGN_32(cNewSize, 4) == cNewSize); if (RT_UNLIKELY(cNewSize > UINT16_MAX)) { rc = RTR0MemObjFree(hMapObj, false /* fFreeMappings (NA) */); AssertRC(rc); return VERR_GMM_TOO_MANY_CHUNK_MAPPINGS; } void *pvMappings = RTMemRealloc(pChunk->paMappingsX, cNewSize * sizeof(pChunk->paMappingsX[0])); if (RT_UNLIKELY(!pvMappings)) { rc = RTR0MemObjFree(hMapObj, false /* fFreeMappings (NA) */); AssertRC(rc); return VERR_NO_MEMORY; } pChunk->paMappingsX = (PGMMCHUNKMAP)pvMappings; } /* insert new entry */ pChunk->paMappingsX[iMapping].hMapObj = hMapObj; pChunk->paMappingsX[iMapping].pGVM = pGVM; Assert(pChunk->cMappingsX == iMapping); pChunk->cMappingsX = iMapping + 1; *ppvR3 = RTR0MemObjAddressR3(hMapObj); } return rc; } /** * Maps a chunk into the user address space of the current process. * * @returns VBox status code. * @param pGMM Pointer to the GMM instance data. * @param pGVM Pointer to the Global VM structure. * @param pChunk Pointer to the chunk to be mapped. * @param fRelaxedSem Whether we can release the semaphore while doing the * mapping (@c true) or not. * @param ppvR3 Where to store the ring-3 address of the mapping. * In the VERR_GMM_CHUNK_ALREADY_MAPPED case, this will be * contain the address of the existing mapping. */ static int gmmR0MapChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, bool fRelaxedSem, PRTR3PTR ppvR3) { /* * Take the chunk lock and leave the giant GMM lock when possible, then * call the worker function. */ GMMR0CHUNKMTXSTATE MtxState; int rc = gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, fRelaxedSem ? GMMR0CHUNK_MTX_RETAKE_GIANT : GMMR0CHUNK_MTX_KEEP_GIANT); if (RT_SUCCESS(rc)) { rc = gmmR0MapChunkLocked(pGMM, pGVM, pChunk, ppvR3); gmmR0ChunkMutexRelease(&MtxState, pChunk); } return rc; } #if defined(VBOX_WITH_PAGE_SHARING) || defined(VBOX_STRICT) /** * Check if a chunk is mapped into the specified VM * * @returns mapped yes/no * @param pGMM Pointer to the GMM instance. * @param pGVM Pointer to the Global VM structure. * @param pChunk Pointer to the chunk to be mapped. * @param ppvR3 Where to store the ring-3 address of the mapping. */ static bool gmmR0IsChunkMapped(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, PRTR3PTR ppvR3) { GMMR0CHUNKMTXSTATE MtxState; gmmR0ChunkMutexAcquire(&MtxState, pGMM, pChunk, GMMR0CHUNK_MTX_KEEP_GIANT); for (uint32_t i = 0; i < pChunk->cMappingsX; i++) { Assert(pChunk->paMappingsX[i].pGVM && pChunk->paMappingsX[i].hMapObj != NIL_RTR0MEMOBJ); if (pChunk->paMappingsX[i].pGVM == pGVM) { *ppvR3 = RTR0MemObjAddressR3(pChunk->paMappingsX[i].hMapObj); gmmR0ChunkMutexRelease(&MtxState, pChunk); return true; } } *ppvR3 = NULL; gmmR0ChunkMutexRelease(&MtxState, pChunk); return false; } #endif /* VBOX_WITH_PAGE_SHARING || VBOX_STRICT */ /** * Map a chunk and/or unmap another chunk. * * The mapping and unmapping applies to the current process. * * This API does two things because it saves a kernel call per mapping when * when the ring-3 mapping cache is full. * * @returns VBox status code. * @param pGVM The global (ring-0) VM structure. * @param idChunkMap The chunk to map. NIL_GMM_CHUNKID if nothing to map. * @param idChunkUnmap The chunk to unmap. NIL_GMM_CHUNKID if nothing to unmap. * @param ppvR3 Where to store the address of the mapped chunk. NULL is ok if nothing to map. * @thread EMT ??? */ GMMR0DECL(int) GMMR0MapUnmapChunk(PGVM pGVM, uint32_t idChunkMap, uint32_t idChunkUnmap, PRTR3PTR ppvR3) { LogFlow(("GMMR0MapUnmapChunk: pGVM=%p idChunkMap=%#x idChunkUnmap=%#x ppvR3=%p\n", pGVM, idChunkMap, idChunkUnmap, ppvR3)); /* * Validate input and get the basics. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVM(pGVM); if (RT_FAILURE(rc)) return rc; AssertCompile(NIL_GMM_CHUNKID == 0); AssertMsgReturn(idChunkMap <= GMM_CHUNKID_LAST, ("%#x\n", idChunkMap), VERR_INVALID_PARAMETER); AssertMsgReturn(idChunkUnmap <= GMM_CHUNKID_LAST, ("%#x\n", idChunkUnmap), VERR_INVALID_PARAMETER); if ( idChunkMap == NIL_GMM_CHUNKID && idChunkUnmap == NIL_GMM_CHUNKID) return VERR_INVALID_PARAMETER; if (idChunkMap != NIL_GMM_CHUNKID) { AssertPtrReturn(ppvR3, VERR_INVALID_POINTER); *ppvR3 = NIL_RTR3PTR; } /* * Take the semaphore and do the work. * * The unmapping is done last since it's easier to undo a mapping than * undoing an unmapping. The ring-3 mapping cache cannot not be so big * that it pushes the user virtual address space to within a chunk of * it it's limits, so, no problem here. */ gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { PGMMCHUNK pMap = NULL; if (idChunkMap != NIL_GVM_HANDLE) { pMap = gmmR0GetChunk(pGMM, idChunkMap); if (RT_LIKELY(pMap)) rc = gmmR0MapChunk(pGMM, pGVM, pMap, true /*fRelaxedSem*/, ppvR3); else { Log(("GMMR0MapUnmapChunk: idChunkMap=%#x\n", idChunkMap)); rc = VERR_GMM_CHUNK_NOT_FOUND; } } /** @todo split this operation, the bail out might (theoretcially) not be * entirely safe. */ if ( idChunkUnmap != NIL_GMM_CHUNKID && RT_SUCCESS(rc)) { PGMMCHUNK pUnmap = gmmR0GetChunk(pGMM, idChunkUnmap); if (RT_LIKELY(pUnmap)) rc = gmmR0UnmapChunk(pGMM, pGVM, pUnmap, true /*fRelaxedSem*/); else { Log(("GMMR0MapUnmapChunk: idChunkUnmap=%#x\n", idChunkUnmap)); rc = VERR_GMM_CHUNK_NOT_FOUND; } if (RT_FAILURE(rc) && pMap) gmmR0UnmapChunk(pGMM, pGVM, pMap, false /*fRelaxedSem*/); } GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); LogFlow(("GMMR0MapUnmapChunk: returns %Rrc\n", rc)); return rc; } /** * VMMR0 request wrapper for GMMR0MapUnmapChunk. * * @returns see GMMR0MapUnmapChunk. * @param pGVM The global (ring-0) VM structure. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0MapUnmapChunkReq(PGVM pGVM, PGMMMAPUNMAPCHUNKREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER); return GMMR0MapUnmapChunk(pGVM, pReq->idChunkMap, pReq->idChunkUnmap, &pReq->pvR3); } #ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM /** * Gets the ring-0 virtual address for the given page. * * This is used by PGM when IEM and such wants to access guest RAM from ring-0. * One of the ASSUMPTIONS here is that the @a idPage is used by the VM and the * corresponding chunk will remain valid beyond the call (at least till the EMT * returns to ring-3). * * @returns VBox status code. * @param pGVM Pointer to the kernel-only VM instace data. * @param idPage The page ID. * @param ppv Where to store the address. * @thread EMT */ GMMR0DECL(int) GMMR0PageIdToVirt(PGVM pGVM, uint32_t idPage, void **ppv) { *ppv = NULL; PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); uint32_t const idChunk = idPage >> GMM_CHUNKID_SHIFT; /* * Start with the per-VM TLB. */ RTSpinlockAcquire(pGVM->gmm.s.hChunkTlbSpinLock); PGMMPERVMCHUNKTLBE pTlbe = &pGVM->gmm.s.aChunkTlbEntries[GMMPERVM_CHUNKTLB_IDX(idChunk)]; PGMMCHUNK pChunk = pTlbe->pChunk; if ( pChunk != NULL && pTlbe->idGeneration == ASMAtomicUoReadU64(&pGMM->idFreeGeneration) && pChunk->Core.Key == idChunk) pGVM->R0Stats.gmm.cChunkTlbHits++; /* hopefully this is a likely outcome */ else { pGVM->R0Stats.gmm.cChunkTlbMisses++; /* * Look it up in the chunk tree. */ RTSpinlockAcquire(pGMM->hSpinLockTree); pChunk = gmmR0GetChunkLocked(pGMM, idChunk); if (RT_LIKELY(pChunk)) { pTlbe->idGeneration = pGMM->idFreeGeneration; RTSpinlockRelease(pGMM->hSpinLockTree); pTlbe->pChunk = pChunk; } else { RTSpinlockRelease(pGMM->hSpinLockTree); RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock); AssertMsgFailed(("idPage=%#x\n", idPage)); return VERR_GMM_PAGE_NOT_FOUND; } } RTSpinlockRelease(pGVM->gmm.s.hChunkTlbSpinLock); /* * Got a chunk, now validate the page ownership and calcuate it's address. */ const GMMPAGE * const pPage = &pChunk->aPages[idPage & GMM_PAGEID_IDX_MASK]; if (RT_LIKELY( ( GMM_PAGE_IS_PRIVATE(pPage) && pPage->Private.hGVM == pGVM->hSelf) || GMM_PAGE_IS_SHARED(pPage))) { AssertPtr(pChunk->pbMapping); *ppv = &pChunk->pbMapping[(idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT]; return VINF_SUCCESS; } AssertMsgFailed(("idPage=%#x is-private=%RTbool Private.hGVM=%u pGVM->hGVM=%u\n", idPage, GMM_PAGE_IS_PRIVATE(pPage), pPage->Private.hGVM, pGVM->hSelf)); return VERR_GMM_NOT_PAGE_OWNER; } #endif /* !VBOX_WITH_LINEAR_HOST_PHYS_MEM */ #ifdef VBOX_WITH_PAGE_SHARING # ifdef VBOX_STRICT /** * For checksumming shared pages in strict builds. * * The purpose is making sure that a page doesn't change. * * @returns Checksum, 0 on failure. * @param pGMM The GMM instance data. * @param pGVM Pointer to the kernel-only VM instace data. * @param idPage The page ID. */ static uint32_t gmmR0StrictPageChecksum(PGMM pGMM, PGVM pGVM, uint32_t idPage) { PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT); AssertMsgReturn(pChunk, ("idPage=%#x\n", idPage), 0); uint8_t *pbChunk; if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk)) return 0; uint8_t const *pbPage = pbChunk + ((idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT); return RTCrc32(pbPage, GUEST_PAGE_SIZE); } # endif /* VBOX_STRICT */ /** * Calculates the module hash value. * * @returns Hash value. * @param pszModuleName The module name. * @param pszVersion The module version string. */ static uint32_t gmmR0ShModCalcHash(const char *pszModuleName, const char *pszVersion) { return RTStrHash1ExN(3, pszModuleName, RTSTR_MAX, "::", (size_t)2, pszVersion, RTSTR_MAX); } /** * Finds a global module. * * @returns Pointer to the global module on success, NULL if not found. * @param pGMM The GMM instance data. * @param uHash The hash as calculated by gmmR0ShModCalcHash. * @param cbModule The module size. * @param enmGuestOS The guest OS type. * @param cRegions The number of regions. * @param pszModuleName The module name. * @param pszVersion The module version. * @param paRegions The region descriptions. */ static PGMMSHAREDMODULE gmmR0ShModFindGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS, uint32_t cRegions, const char *pszModuleName, const char *pszVersion, struct VMMDEVSHAREDREGIONDESC const *paRegions) { for (PGMMSHAREDMODULE pGblMod = (PGMMSHAREDMODULE)RTAvllU32Get(&pGMM->pGlobalSharedModuleTree, uHash); pGblMod; pGblMod = (PGMMSHAREDMODULE)pGblMod->Core.pList) { if (pGblMod->cbModule != cbModule) continue; if (pGblMod->enmGuestOS != enmGuestOS) continue; if (pGblMod->cRegions != cRegions) continue; if (strcmp(pGblMod->szName, pszModuleName)) continue; if (strcmp(pGblMod->szVersion, pszVersion)) continue; uint32_t i; for (i = 0; i < cRegions; i++) { uint32_t off = paRegions[i].GCRegionAddr & GUEST_PAGE_OFFSET_MASK; if (pGblMod->aRegions[i].off != off) break; uint32_t cb = RT_ALIGN_32(paRegions[i].cbRegion + off, GUEST_PAGE_SIZE); if (pGblMod->aRegions[i].cb != cb) break; } if (i == cRegions) return pGblMod; } return NULL; } /** * Creates a new global module. * * @returns VBox status code. * @param pGMM The GMM instance data. * @param uHash The hash as calculated by gmmR0ShModCalcHash. * @param cbModule The module size. * @param enmGuestOS The guest OS type. * @param cRegions The number of regions. * @param pszModuleName The module name. * @param pszVersion The module version. * @param paRegions The region descriptions. * @param ppGblMod Where to return the new module on success. */ static int gmmR0ShModNewGlobal(PGMM pGMM, uint32_t uHash, uint32_t cbModule, VBOXOSFAMILY enmGuestOS, uint32_t cRegions, const char *pszModuleName, const char *pszVersion, struct VMMDEVSHAREDREGIONDESC const *paRegions, PGMMSHAREDMODULE *ppGblMod) { Log(("gmmR0ShModNewGlobal: %s %s size %#x os %u rgn %u\n", pszModuleName, pszVersion, cbModule, enmGuestOS, cRegions)); if (pGMM->cShareableModules >= GMM_MAX_SHARED_GLOBAL_MODULES) { Log(("gmmR0ShModNewGlobal: Too many modules\n")); return VERR_GMM_TOO_MANY_GLOBAL_MODULES; } PGMMSHAREDMODULE pGblMod = (PGMMSHAREDMODULE)RTMemAllocZ(RT_UOFFSETOF_DYN(GMMSHAREDMODULE, aRegions[cRegions])); if (!pGblMod) { Log(("gmmR0ShModNewGlobal: No memory\n")); return VERR_NO_MEMORY; } pGblMod->Core.Key = uHash; pGblMod->cbModule = cbModule; pGblMod->cRegions = cRegions; pGblMod->cUsers = 1; pGblMod->enmGuestOS = enmGuestOS; strcpy(pGblMod->szName, pszModuleName); strcpy(pGblMod->szVersion, pszVersion); for (uint32_t i = 0; i < cRegions; i++) { Log(("gmmR0ShModNewGlobal: rgn[%u]=%RGvLB%#x\n", i, paRegions[i].GCRegionAddr, paRegions[i].cbRegion)); pGblMod->aRegions[i].off = paRegions[i].GCRegionAddr & GUEST_PAGE_OFFSET_MASK; pGblMod->aRegions[i].cb = paRegions[i].cbRegion + pGblMod->aRegions[i].off; pGblMod->aRegions[i].cb = RT_ALIGN_32(pGblMod->aRegions[i].cb, GUEST_PAGE_SIZE); pGblMod->aRegions[i].paidPages = NULL; /* allocated when needed. */ } bool fInsert = RTAvllU32Insert(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core); Assert(fInsert); NOREF(fInsert); pGMM->cShareableModules++; *ppGblMod = pGblMod; return VINF_SUCCESS; } /** * Deletes a global module which is no longer referenced by anyone. * * @param pGMM The GMM instance data. * @param pGblMod The module to delete. */ static void gmmR0ShModDeleteGlobal(PGMM pGMM, PGMMSHAREDMODULE pGblMod) { Assert(pGblMod->cUsers == 0); Assert(pGMM->cShareableModules > 0 && pGMM->cShareableModules <= GMM_MAX_SHARED_GLOBAL_MODULES); void *pvTest = RTAvllU32RemoveNode(&pGMM->pGlobalSharedModuleTree, &pGblMod->Core); Assert(pvTest == pGblMod); NOREF(pvTest); pGMM->cShareableModules--; uint32_t i = pGblMod->cRegions; while (i-- > 0) { if (pGblMod->aRegions[i].paidPages) { /* We don't doing anything to the pages as they are handled by the copy-on-write mechanism in PGM. */ RTMemFree(pGblMod->aRegions[i].paidPages); pGblMod->aRegions[i].paidPages = NULL; } } RTMemFree(pGblMod); } static int gmmR0ShModNewPerVM(PGVM pGVM, RTGCPTR GCBaseAddr, uint32_t cRegions, const VMMDEVSHAREDREGIONDESC *paRegions, PGMMSHAREDMODULEPERVM *ppRecVM) { if (pGVM->gmm.s.Stats.cShareableModules >= GMM_MAX_SHARED_PER_VM_MODULES) return VERR_GMM_TOO_MANY_PER_VM_MODULES; PGMMSHAREDMODULEPERVM pRecVM; pRecVM = (PGMMSHAREDMODULEPERVM)RTMemAllocZ(RT_UOFFSETOF_DYN(GMMSHAREDMODULEPERVM, aRegionsGCPtrs[cRegions])); if (!pRecVM) return VERR_NO_MEMORY; pRecVM->Core.Key = GCBaseAddr; for (uint32_t i = 0; i < cRegions; i++) pRecVM->aRegionsGCPtrs[i] = paRegions[i].GCRegionAddr; bool fInsert = RTAvlGCPtrInsert(&pGVM->gmm.s.pSharedModuleTree, &pRecVM->Core); Assert(fInsert); NOREF(fInsert); pGVM->gmm.s.Stats.cShareableModules++; *ppRecVM = pRecVM; return VINF_SUCCESS; } static void gmmR0ShModDeletePerVM(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULEPERVM pRecVM, bool fRemove) { /* * Free the per-VM module. */ PGMMSHAREDMODULE pGblMod = pRecVM->pGlobalModule; pRecVM->pGlobalModule = NULL; if (fRemove) { void *pvTest = RTAvlGCPtrRemove(&pGVM->gmm.s.pSharedModuleTree, pRecVM->Core.Key); Assert(pvTest == &pRecVM->Core); NOREF(pvTest); } RTMemFree(pRecVM); /* * Release the global module. * (In the registration bailout case, it might not be.) */ if (pGblMod) { Assert(pGblMod->cUsers > 0); pGblMod->cUsers--; if (pGblMod->cUsers == 0) gmmR0ShModDeleteGlobal(pGMM, pGblMod); } } #endif /* VBOX_WITH_PAGE_SHARING */ /** * Registers a new shared module for the VM. * * @returns VBox status code. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param enmGuestOS The guest OS type. * @param pszModuleName The module name. * @param pszVersion The module version. * @param GCPtrModBase The module base address. * @param cbModule The module size. * @param cRegions The mumber of shared region descriptors. * @param paRegions Pointer to an array of shared region(s). * @thread EMT(idCpu) */ GMMR0DECL(int) GMMR0RegisterSharedModule(PGVM pGVM, VMCPUID idCpu, VBOXOSFAMILY enmGuestOS, char *pszModuleName, char *pszVersion, RTGCPTR GCPtrModBase, uint32_t cbModule, uint32_t cRegions, struct VMMDEVSHAREDREGIONDESC const *paRegions) { #ifdef VBOX_WITH_PAGE_SHARING /* * Validate input and get the basics. * * Note! Turns out the module size does necessarily match the size of the * regions. (iTunes on XP) */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; if (RT_UNLIKELY(cRegions > VMMDEVSHAREDREGIONDESC_MAX)) return VERR_GMM_TOO_MANY_REGIONS; if (RT_UNLIKELY(cbModule == 0 || cbModule > _1G)) return VERR_GMM_BAD_SHARED_MODULE_SIZE; uint32_t cbTotal = 0; for (uint32_t i = 0; i < cRegions; i++) { if (RT_UNLIKELY(paRegions[i].cbRegion == 0 || paRegions[i].cbRegion > _1G)) return VERR_GMM_SHARED_MODULE_BAD_REGIONS_SIZE; cbTotal += paRegions[i].cbRegion; if (RT_UNLIKELY(cbTotal > _1G)) return VERR_GMM_SHARED_MODULE_BAD_REGIONS_SIZE; } AssertPtrReturn(pszModuleName, VERR_INVALID_POINTER); if (RT_UNLIKELY(!memchr(pszModuleName, '\0', GMM_SHARED_MODULE_MAX_NAME_STRING))) return VERR_GMM_MODULE_NAME_TOO_LONG; AssertPtrReturn(pszVersion, VERR_INVALID_POINTER); if (RT_UNLIKELY(!memchr(pszVersion, '\0', GMM_SHARED_MODULE_MAX_VERSION_STRING))) return VERR_GMM_MODULE_NAME_TOO_LONG; uint32_t const uHash = gmmR0ShModCalcHash(pszModuleName, pszVersion); Log(("GMMR0RegisterSharedModule %s %s base %RGv size %x hash %x\n", pszModuleName, pszVersion, GCPtrModBase, cbModule, uHash)); /* * Take the semaphore and do some more validations. */ gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { /* * Check if this module is already locally registered and register * it if it isn't. The base address is a unique module identifier * locally. */ PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)RTAvlGCPtrGet(&pGVM->gmm.s.pSharedModuleTree, GCPtrModBase); bool fNewModule = pRecVM == NULL; if (fNewModule) { rc = gmmR0ShModNewPerVM(pGVM, GCPtrModBase, cRegions, paRegions, &pRecVM); if (RT_SUCCESS(rc)) { /* * Find a matching global module, register a new one if needed. */ PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions, pszModuleName, pszVersion, paRegions); if (!pGblMod) { Assert(fNewModule); rc = gmmR0ShModNewGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions, pszModuleName, pszVersion, paRegions, &pGblMod); if (RT_SUCCESS(rc)) { pRecVM->pGlobalModule = pGblMod; /* (One referenced returned by gmmR0ShModNewGlobal.) */ Log(("GMMR0RegisterSharedModule: new module %s %s\n", pszModuleName, pszVersion)); } else gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/); } else { Assert(pGblMod->cUsers > 0 && pGblMod->cUsers < UINT32_MAX / 2); pGblMod->cUsers++; pRecVM->pGlobalModule = pGblMod; Log(("GMMR0RegisterSharedModule: new per vm module %s %s, gbl users %d\n", pszModuleName, pszVersion, pGblMod->cUsers)); } } } else { /* * Attempt to re-register an existing module. */ PGMMSHAREDMODULE pGblMod = gmmR0ShModFindGlobal(pGMM, uHash, cbModule, enmGuestOS, cRegions, pszModuleName, pszVersion, paRegions); if (pRecVM->pGlobalModule == pGblMod) { Log(("GMMR0RegisterSharedModule: already registered %s %s, gbl users %d\n", pszModuleName, pszVersion, pGblMod->cUsers)); rc = VINF_GMM_SHARED_MODULE_ALREADY_REGISTERED; } else { /** @todo may have to unregister+register when this happens in case it's caused * by VBoxService crashing and being restarted... */ Log(("GMMR0RegisterSharedModule: Address clash!\n" " incoming at %RGvLB%#x %s %s rgns %u\n" " existing at %RGvLB%#x %s %s rgns %u\n", GCPtrModBase, cbModule, pszModuleName, pszVersion, cRegions, pRecVM->Core.Key, pRecVM->pGlobalModule->cbModule, pRecVM->pGlobalModule->szName, pRecVM->pGlobalModule->szVersion, pRecVM->pGlobalModule->cRegions)); rc = VERR_GMM_SHARED_MODULE_ADDRESS_CLASH; } } GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); return rc; #else NOREF(pGVM); NOREF(idCpu); NOREF(enmGuestOS); NOREF(pszModuleName); NOREF(pszVersion); NOREF(GCPtrModBase); NOREF(cbModule); NOREF(cRegions); NOREF(paRegions); return VERR_NOT_IMPLEMENTED; #endif } /** * VMMR0 request wrapper for GMMR0RegisterSharedModule. * * @returns see GMMR0RegisterSharedModule. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0RegisterSharedModuleReq(PGVM pGVM, VMCPUID idCpu, PGMMREGISTERSHAREDMODULEREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn( pReq->Hdr.cbReq >= sizeof(*pReq) && pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(GMMREGISTERSHAREDMODULEREQ, aRegions[pReq->cRegions]), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER); /* Pass back return code in the request packet to preserve informational codes. (VMMR3CallR0 chokes on them) */ pReq->rc = GMMR0RegisterSharedModule(pGVM, idCpu, pReq->enmGuestOS, pReq->szName, pReq->szVersion, pReq->GCBaseAddr, pReq->cbModule, pReq->cRegions, pReq->aRegions); return VINF_SUCCESS; } /** * Unregisters a shared module for the VM * * @returns VBox status code. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param pszModuleName The module name. * @param pszVersion The module version. * @param GCPtrModBase The module base address. * @param cbModule The module size. */ GMMR0DECL(int) GMMR0UnregisterSharedModule(PGVM pGVM, VMCPUID idCpu, char *pszModuleName, char *pszVersion, RTGCPTR GCPtrModBase, uint32_t cbModule) { #ifdef VBOX_WITH_PAGE_SHARING /* * Validate input and get the basics. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; AssertPtrReturn(pszModuleName, VERR_INVALID_POINTER); AssertPtrReturn(pszVersion, VERR_INVALID_POINTER); if (RT_UNLIKELY(!memchr(pszModuleName, '\0', GMM_SHARED_MODULE_MAX_NAME_STRING))) return VERR_GMM_MODULE_NAME_TOO_LONG; if (RT_UNLIKELY(!memchr(pszVersion, '\0', GMM_SHARED_MODULE_MAX_VERSION_STRING))) return VERR_GMM_MODULE_NAME_TOO_LONG; Log(("GMMR0UnregisterSharedModule %s %s base=%RGv size %x\n", pszModuleName, pszVersion, GCPtrModBase, cbModule)); /* * Take the semaphore and do some more validations. */ gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { /* * Locate and remove the specified module. */ PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)RTAvlGCPtrGet(&pGVM->gmm.s.pSharedModuleTree, GCPtrModBase); if (pRecVM) { /** @todo Do we need to do more validations here, like that the * name + version + cbModule matches? */ NOREF(cbModule); Assert(pRecVM->pGlobalModule); gmmR0ShModDeletePerVM(pGMM, pGVM, pRecVM, true /*fRemove*/); } else rc = VERR_GMM_SHARED_MODULE_NOT_FOUND; GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); return rc; #else NOREF(pGVM); NOREF(idCpu); NOREF(pszModuleName); NOREF(pszVersion); NOREF(GCPtrModBase); NOREF(cbModule); return VERR_NOT_IMPLEMENTED; #endif } /** * VMMR0 request wrapper for GMMR0UnregisterSharedModule. * * @returns see GMMR0UnregisterSharedModule. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0UnregisterSharedModuleReq(PGVM pGVM, VMCPUID idCpu, PGMMUNREGISTERSHAREDMODULEREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER); return GMMR0UnregisterSharedModule(pGVM, idCpu, pReq->szName, pReq->szVersion, pReq->GCBaseAddr, pReq->cbModule); } #ifdef VBOX_WITH_PAGE_SHARING /** * Increase the use count of a shared page, the page is known to exist and be valid and such. * * @param pGMM Pointer to the GMM instance. * @param pGVM Pointer to the GVM instance. * @param pPage The page structure. */ DECLINLINE(void) gmmR0UseSharedPage(PGMM pGMM, PGVM pGVM, PGMMPAGE pPage) { Assert(pGMM->cSharedPages > 0); Assert(pGMM->cAllocatedPages > 0); pGMM->cDuplicatePages++; pPage->Shared.cRefs++; pGVM->gmm.s.Stats.cSharedPages++; pGVM->gmm.s.Stats.Allocated.cBasePages++; } /** * Converts a private page to a shared page, the page is known to exist and be valid and such. * * @param pGMM Pointer to the GMM instance. * @param pGVM Pointer to the GVM instance. * @param HCPhys Host physical address * @param idPage The Page ID * @param pPage The page structure. * @param pPageDesc Shared page descriptor */ DECLINLINE(void) gmmR0ConvertToSharedPage(PGMM pGMM, PGVM pGVM, RTHCPHYS HCPhys, uint32_t idPage, PGMMPAGE pPage, PGMMSHAREDPAGEDESC pPageDesc) { PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, idPage >> GMM_CHUNKID_SHIFT); Assert(pChunk); Assert(pChunk->cFree < GMM_CHUNK_NUM_PAGES); Assert(GMM_PAGE_IS_PRIVATE(pPage)); pChunk->cPrivate--; pChunk->cShared++; pGMM->cSharedPages++; pGVM->gmm.s.Stats.cSharedPages++; pGVM->gmm.s.Stats.cPrivatePages--; /* Modify the page structure. */ pPage->Shared.pfn = (uint32_t)(uint64_t)(HCPhys >> GUEST_PAGE_SHIFT); pPage->Shared.cRefs = 1; #ifdef VBOX_STRICT pPageDesc->u32StrictChecksum = gmmR0StrictPageChecksum(pGMM, pGVM, idPage); pPage->Shared.u14Checksum = pPageDesc->u32StrictChecksum; #else NOREF(pPageDesc); pPage->Shared.u14Checksum = 0; #endif pPage->Shared.u2State = GMM_PAGE_STATE_SHARED; } static int gmmR0SharedModuleCheckPageFirstTime(PGMM pGMM, PGVM pGVM, PGMMSHAREDMODULE pModule, unsigned idxRegion, unsigned idxPage, PGMMSHAREDPAGEDESC pPageDesc, PGMMSHAREDREGIONDESC pGlobalRegion) { NOREF(pModule); /* Easy case: just change the internal page type. */ PGMMPAGE pPage = gmmR0GetPage(pGMM, pPageDesc->idPage); AssertMsgReturn(pPage, ("idPage=%#x (GCPhys=%RGp HCPhys=%RHp idxRegion=%#x idxPage=%#x) #1\n", pPageDesc->idPage, pPageDesc->GCPhys, pPageDesc->HCPhys, idxRegion, idxPage), VERR_PGM_PHYS_INVALID_PAGE_ID); NOREF(idxRegion); AssertMsg(pPageDesc->GCPhys == (pPage->Private.pfn << 12), ("desc %RGp gmm %RGp\n", pPageDesc->HCPhys, (pPage->Private.pfn << 12))); gmmR0ConvertToSharedPage(pGMM, pGVM, pPageDesc->HCPhys, pPageDesc->idPage, pPage, pPageDesc); /* Keep track of these references. */ pGlobalRegion->paidPages[idxPage] = pPageDesc->idPage; return VINF_SUCCESS; } /** * Checks specified shared module range for changes * * Performs the following tasks: * - If a shared page is new, then it changes the GMM page type to shared and * returns it in the pPageDesc descriptor. * - If a shared page already exists, then it checks if the VM page is * identical and if so frees the VM page and returns the shared page in * pPageDesc descriptor. * * @remarks ASSUMES the caller has acquired the GMM semaphore!! * * @returns VBox status code. * @param pGVM Pointer to the GVM instance data. * @param pModule Module description * @param idxRegion Region index * @param idxPage Page index * @param pPageDesc Page descriptor */ GMMR0DECL(int) GMMR0SharedModuleCheckPage(PGVM pGVM, PGMMSHAREDMODULE pModule, uint32_t idxRegion, uint32_t idxPage, PGMMSHAREDPAGEDESC pPageDesc) { int rc; PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); pPageDesc->u32StrictChecksum = 0; AssertMsgReturn(idxRegion < pModule->cRegions, ("idxRegion=%#x cRegions=%#x %s %s\n", idxRegion, pModule->cRegions, pModule->szName, pModule->szVersion), VERR_INVALID_PARAMETER); uint32_t const cPages = pModule->aRegions[idxRegion].cb >> GUEST_PAGE_SHIFT; AssertMsgReturn(idxPage < cPages, ("idxRegion=%#x cRegions=%#x %s %s\n", idxRegion, pModule->cRegions, pModule->szName, pModule->szVersion), VERR_INVALID_PARAMETER); LogFlow(("GMMR0SharedModuleCheckRange %s base %RGv region %d idxPage %d\n", pModule->szName, pModule->Core.Key, idxRegion, idxPage)); /* * First time; create a page descriptor array. */ PGMMSHAREDREGIONDESC pGlobalRegion = &pModule->aRegions[idxRegion]; if (!pGlobalRegion->paidPages) { Log(("Allocate page descriptor array for %d pages\n", cPages)); pGlobalRegion->paidPages = (uint32_t *)RTMemAlloc(cPages * sizeof(pGlobalRegion->paidPages[0])); AssertReturn(pGlobalRegion->paidPages, VERR_NO_MEMORY); /* Invalidate all descriptors. */ uint32_t i = cPages; while (i-- > 0) pGlobalRegion->paidPages[i] = NIL_GMM_PAGEID; } /* * We've seen this shared page for the first time? */ if (pGlobalRegion->paidPages[idxPage] == NIL_GMM_PAGEID) { Log(("New shared page guest %RGp host %RHp\n", pPageDesc->GCPhys, pPageDesc->HCPhys)); return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion); } /* * We've seen it before... */ Log(("Replace existing page guest %RGp host %RHp id %#x -> id %#x\n", pPageDesc->GCPhys, pPageDesc->HCPhys, pPageDesc->idPage, pGlobalRegion->paidPages[idxPage])); Assert(pPageDesc->idPage != pGlobalRegion->paidPages[idxPage]); /* * Get the shared page source. */ PGMMPAGE pPage = gmmR0GetPage(pGMM, pGlobalRegion->paidPages[idxPage]); AssertMsgReturn(pPage, ("idPage=%#x (idxRegion=%#x idxPage=%#x) #2\n", pPageDesc->idPage, idxRegion, idxPage), VERR_PGM_PHYS_INVALID_PAGE_ID); if (pPage->Common.u2State != GMM_PAGE_STATE_SHARED) { /* * Page was freed at some point; invalidate this entry. */ /** @todo this isn't really bullet proof. */ Log(("Old shared page was freed -> create a new one\n")); pGlobalRegion->paidPages[idxPage] = NIL_GMM_PAGEID; return gmmR0SharedModuleCheckPageFirstTime(pGMM, pGVM, pModule, idxRegion, idxPage, pPageDesc, pGlobalRegion); } Log(("Replace existing page guest host %RHp -> %RHp\n", pPageDesc->HCPhys, ((uint64_t)pPage->Shared.pfn) << GUEST_PAGE_SHIFT)); /* * Calculate the virtual address of the local page. */ PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pPageDesc->idPage >> GMM_CHUNKID_SHIFT); AssertMsgReturn(pChunk, ("idPage=%#x (idxRegion=%#x idxPage=%#x) #4\n", pPageDesc->idPage, idxRegion, idxPage), VERR_PGM_PHYS_INVALID_PAGE_ID); uint8_t *pbChunk; AssertMsgReturn(gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk), ("idPage=%#x (idxRegion=%#x idxPage=%#x) #3\n", pPageDesc->idPage, idxRegion, idxPage), VERR_PGM_PHYS_INVALID_PAGE_ID); uint8_t *pbLocalPage = pbChunk + ((pPageDesc->idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT); /* * Calculate the virtual address of the shared page. */ pChunk = gmmR0GetChunk(pGMM, pGlobalRegion->paidPages[idxPage] >> GMM_CHUNKID_SHIFT); Assert(pChunk); /* can't fail as gmmR0GetPage succeeded. */ /* * Get the virtual address of the physical page; map the chunk into the VM * process if not already done. */ if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk)) { Log(("Map chunk into process!\n")); rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk); AssertRCReturn(rc, rc); } uint8_t *pbSharedPage = pbChunk + ((pGlobalRegion->paidPages[idxPage] & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT); #ifdef VBOX_STRICT pPageDesc->u32StrictChecksum = RTCrc32(pbSharedPage, GUEST_PAGE_SIZE); uint32_t uChecksum = pPageDesc->u32StrictChecksum & UINT32_C(0x00003fff); AssertMsg(!uChecksum || uChecksum == pPage->Shared.u14Checksum || !pPage->Shared.u14Checksum, ("%#x vs %#x - idPage=%#x - %s %s\n", uChecksum, pPage->Shared.u14Checksum, pGlobalRegion->paidPages[idxPage], pModule->szName, pModule->szVersion)); #endif if (memcmp(pbSharedPage, pbLocalPage, GUEST_PAGE_SIZE)) { Log(("Unexpected differences found between local and shared page; skip\n")); /* Signal to the caller that this one hasn't changed. */ pPageDesc->idPage = NIL_GMM_PAGEID; return VINF_SUCCESS; } /* * Free the old local page. */ GMMFREEPAGEDESC PageDesc; PageDesc.idPage = pPageDesc->idPage; rc = gmmR0FreePages(pGMM, pGVM, 1, &PageDesc, GMMACCOUNT_BASE); AssertRCReturn(rc, rc); gmmR0UseSharedPage(pGMM, pGVM, pPage); /* * Pass along the new physical address & page id. */ pPageDesc->HCPhys = ((uint64_t)pPage->Shared.pfn) << GUEST_PAGE_SHIFT; pPageDesc->idPage = pGlobalRegion->paidPages[idxPage]; return VINF_SUCCESS; } /** * RTAvlGCPtrDestroy callback. * * @returns 0 or VERR_GMM_INSTANCE. * @param pNode The node to destroy. * @param pvArgs Pointer to an argument packet. */ static DECLCALLBACK(int) gmmR0CleanupSharedModule(PAVLGCPTRNODECORE pNode, void *pvArgs) { gmmR0ShModDeletePerVM(((GMMR0SHMODPERVMDTORARGS *)pvArgs)->pGMM, ((GMMR0SHMODPERVMDTORARGS *)pvArgs)->pGVM, (PGMMSHAREDMODULEPERVM)pNode, false /*fRemove*/); return VINF_SUCCESS; } /** * Used by GMMR0CleanupVM to clean up shared modules. * * This is called without taking the GMM lock so that it can be yielded as * needed here. * * @param pGMM The GMM handle. * @param pGVM The global VM handle. */ static void gmmR0SharedModuleCleanup(PGMM pGMM, PGVM pGVM) { gmmR0MutexAcquire(pGMM); GMM_CHECK_SANITY_UPON_ENTERING(pGMM); GMMR0SHMODPERVMDTORARGS Args; Args.pGVM = pGVM; Args.pGMM = pGMM; RTAvlGCPtrDestroy(&pGVM->gmm.s.pSharedModuleTree, gmmR0CleanupSharedModule, &Args); AssertMsg(pGVM->gmm.s.Stats.cShareableModules == 0, ("%d\n", pGVM->gmm.s.Stats.cShareableModules)); pGVM->gmm.s.Stats.cShareableModules = 0; gmmR0MutexRelease(pGMM); } #endif /* VBOX_WITH_PAGE_SHARING */ /** * Removes all shared modules for the specified VM * * @returns VBox status code. * @param pGVM The global (ring-0) VM structure. * @param idCpu The VCPU id. */ GMMR0DECL(int) GMMR0ResetSharedModules(PGVM pGVM, VMCPUID idCpu) { #ifdef VBOX_WITH_PAGE_SHARING /* * Validate input and get the basics. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; /* * Take the semaphore and do some more validations. */ gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { Log(("GMMR0ResetSharedModules\n")); GMMR0SHMODPERVMDTORARGS Args; Args.pGVM = pGVM; Args.pGMM = pGMM; RTAvlGCPtrDestroy(&pGVM->gmm.s.pSharedModuleTree, gmmR0CleanupSharedModule, &Args); pGVM->gmm.s.Stats.cShareableModules = 0; rc = VINF_SUCCESS; GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); return rc; #else RT_NOREF(pGVM, idCpu); return VERR_NOT_IMPLEMENTED; #endif } #ifdef VBOX_WITH_PAGE_SHARING /** * Tree enumeration callback for checking a shared module. */ static DECLCALLBACK(int) gmmR0CheckSharedModule(PAVLGCPTRNODECORE pNode, void *pvUser) { GMMCHECKSHAREDMODULEINFO *pArgs = (GMMCHECKSHAREDMODULEINFO*)pvUser; PGMMSHAREDMODULEPERVM pRecVM = (PGMMSHAREDMODULEPERVM)pNode; PGMMSHAREDMODULE pGblMod = pRecVM->pGlobalModule; Log(("gmmR0CheckSharedModule: check %s %s base=%RGv size=%x\n", pGblMod->szName, pGblMod->szVersion, pGblMod->Core.Key, pGblMod->cbModule)); int rc = PGMR0SharedModuleCheck(pArgs->pGVM, pArgs->pGVM, pArgs->idCpu, pGblMod, pRecVM->aRegionsGCPtrs); if (RT_FAILURE(rc)) return rc; return VINF_SUCCESS; } #endif /* VBOX_WITH_PAGE_SHARING */ /** * Check all shared modules for the specified VM. * * @returns VBox status code. * @param pGVM The global (ring-0) VM structure. * @param idCpu The calling EMT number. * @thread EMT(idCpu) */ GMMR0DECL(int) GMMR0CheckSharedModules(PGVM pGVM, VMCPUID idCpu) { #ifdef VBOX_WITH_PAGE_SHARING /* * Validate input and get the basics. */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu); if (RT_FAILURE(rc)) return rc; # ifndef DEBUG_sandervl /* * Take the semaphore and do some more validations. */ gmmR0MutexAcquire(pGMM); # endif if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { /* * Walk the tree, checking each module. */ Log(("GMMR0CheckSharedModules\n")); GMMCHECKSHAREDMODULEINFO Args; Args.pGVM = pGVM; Args.idCpu = idCpu; rc = RTAvlGCPtrDoWithAll(&pGVM->gmm.s.pSharedModuleTree, true /* fFromLeft */, gmmR0CheckSharedModule, &Args); Log(("GMMR0CheckSharedModules done (rc=%Rrc)!\n", rc)); GMM_CHECK_SANITY_UPON_LEAVING(pGMM); } else rc = VERR_GMM_IS_NOT_SANE; # ifndef DEBUG_sandervl gmmR0MutexRelease(pGMM); # endif return rc; #else RT_NOREF(pGVM, idCpu); return VERR_NOT_IMPLEMENTED; #endif } #ifdef VBOX_STRICT /** * Worker for GMMR0FindDuplicatePageReq. * * @returns true if duplicate, false if not. */ static bool gmmR0FindDupPageInChunk(PGMM pGMM, PGVM pGVM, PGMMCHUNK pChunk, uint8_t const *pbSourcePage) { bool fFoundDuplicate = false; /* Only take chunks not mapped into this VM process; not entirely correct. */ uint8_t *pbChunk; if (!gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk)) { int rc = gmmR0MapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/, (PRTR3PTR)&pbChunk); if (RT_SUCCESS(rc)) { /* * Look for duplicate pages */ uintptr_t iPage = GMM_CHUNK_NUM_PAGES; while (iPage-- > 0) { if (GMM_PAGE_IS_PRIVATE(&pChunk->aPages[iPage])) { uint8_t *pbDestPage = pbChunk + (iPage << GUEST_PAGE_SHIFT); if (!memcmp(pbSourcePage, pbDestPage, GUEST_PAGE_SIZE)) { fFoundDuplicate = true; break; } } } gmmR0UnmapChunk(pGMM, pGVM, pChunk, false /*fRelaxedSem*/); } } return fFoundDuplicate; } /** * Find a duplicate of the specified page in other active VMs * * @returns VBox status code. * @param pGVM The global (ring-0) VM structure. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0FindDuplicatePageReq(PGVM pGVM, PGMMFINDDUPLICATEPAGEREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER); PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); int rc = GVMMR0ValidateGVM(pGVM); if (RT_FAILURE(rc)) return rc; /* * Take the semaphore and do some more validations. */ rc = gmmR0MutexAcquire(pGMM); if (GMM_CHECK_SANITY_UPON_ENTERING(pGMM)) { uint8_t *pbChunk; PGMMCHUNK pChunk = gmmR0GetChunk(pGMM, pReq->idPage >> GMM_CHUNKID_SHIFT); if (pChunk) { if (gmmR0IsChunkMapped(pGMM, pGVM, pChunk, (PRTR3PTR)&pbChunk)) { uint8_t *pbSourcePage = pbChunk + ((pReq->idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT); PGMMPAGE pPage = gmmR0GetPage(pGMM, pReq->idPage); if (pPage) { /* * Walk the chunks */ pReq->fDuplicate = false; RTListForEach(&pGMM->ChunkList, pChunk, GMMCHUNK, ListNode) { if (gmmR0FindDupPageInChunk(pGMM, pGVM, pChunk, pbSourcePage)) { pReq->fDuplicate = true; break; } } } else { AssertFailed(); rc = VERR_PGM_PHYS_INVALID_PAGE_ID; } } else AssertFailed(); } else AssertFailed(); } else rc = VERR_GMM_IS_NOT_SANE; gmmR0MutexRelease(pGMM); return rc; } #endif /* VBOX_STRICT */ /** * Retrieves the GMM statistics visible to the caller. * * @returns VBox status code. * * @param pStats Where to put the statistics. * @param pSession The current session. * @param pGVM The GVM to obtain statistics for. Optional. */ GMMR0DECL(int) GMMR0QueryStatistics(PGMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM) { LogFlow(("GVMMR0QueryStatistics: pStats=%p pSession=%p pGVM=%p\n", pStats, pSession, pGVM)); /* * Validate input. */ AssertPtrReturn(pSession, VERR_INVALID_POINTER); AssertPtrReturn(pStats, VERR_INVALID_POINTER); pStats->cMaxPages = 0; /* (crash before taking the mutex...) */ PGMM pGMM; GMM_GET_VALID_INSTANCE(pGMM, VERR_GMM_INSTANCE); /* * Validate the VM handle, if not NULL, and lock the GMM. */ int rc; if (pGVM) { rc = GVMMR0ValidateGVM(pGVM); if (RT_FAILURE(rc)) return rc; } rc = gmmR0MutexAcquire(pGMM); if (RT_FAILURE(rc)) return rc; /* * Copy out the GMM statistics. */ pStats->cMaxPages = pGMM->cMaxPages; pStats->cReservedPages = pGMM->cReservedPages; pStats->cOverCommittedPages = pGMM->cOverCommittedPages; pStats->cAllocatedPages = pGMM->cAllocatedPages; pStats->cSharedPages = pGMM->cSharedPages; pStats->cDuplicatePages = pGMM->cDuplicatePages; pStats->cLeftBehindSharedPages = pGMM->cLeftBehindSharedPages; pStats->cBalloonedPages = pGMM->cBalloonedPages; pStats->cChunks = pGMM->cChunks; pStats->cFreedChunks = pGMM->cFreedChunks; pStats->cShareableModules = pGMM->cShareableModules; pStats->idFreeGeneration = pGMM->idFreeGeneration; RT_ZERO(pStats->au64Reserved); /* * Copy out the VM statistics. */ if (pGVM) pStats->VMStats = pGVM->gmm.s.Stats; else RT_ZERO(pStats->VMStats); gmmR0MutexRelease(pGMM); return rc; } /** * VMMR0 request wrapper for GMMR0QueryStatistics. * * @returns see GMMR0QueryStatistics. * @param pGVM The global (ring-0) VM structure. Optional. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0QueryStatisticsReq(PGVM pGVM, PGMMQUERYSTATISTICSSREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER); return GMMR0QueryStatistics(&pReq->Stats, pReq->pSession, pGVM); } /** * Resets the specified GMM statistics. * * @returns VBox status code. * * @param pStats Which statistics to reset, that is, non-zero fields * indicates which to reset. * @param pSession The current session. * @param pGVM The GVM to reset statistics for. Optional. */ GMMR0DECL(int) GMMR0ResetStatistics(PCGMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM) { NOREF(pStats); NOREF(pSession); NOREF(pGVM); /* Currently nothing we can reset at the moment. */ return VINF_SUCCESS; } /** * VMMR0 request wrapper for GMMR0ResetStatistics. * * @returns see GMMR0ResetStatistics. * @param pGVM The global (ring-0) VM structure. Optional. * @param pReq Pointer to the request packet. */ GMMR0DECL(int) GMMR0ResetStatisticsReq(PGVM pGVM, PGMMRESETSTATISTICSSREQ pReq) { /* * Validate input and pass it on. */ AssertPtrReturn(pReq, VERR_INVALID_POINTER); AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER); return GMMR0ResetStatistics(&pReq->Stats, pReq->pSession, pGVM); }