VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 96860

Last change on this file since 96860 was 96407, checked in by vboxsync, 22 months ago

scm copyright and license note update
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 224.8 KB
Line 
1/* $Id: PGMPhys.cpp 96407 2022-08-22 17:43:14Z vboxsync $ */
2/** @file
3 * PGM - Page Manager and Monitor, Physical Memory Addressing.
4 */
5
6/*
7 * Copyright (C) 2006-2022 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_PGM_PHYS
33#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34#include <VBox/vmm/pgm.h>
35#include <VBox/vmm/iem.h>
36#include <VBox/vmm/iom.h>
37#include <VBox/vmm/mm.h>
38#include <VBox/vmm/nem.h>
39#include <VBox/vmm/stam.h>
40#include <VBox/vmm/pdmdev.h>
41#include "PGMInternal.h"
42#include <VBox/vmm/vmcc.h>
43
44#include "PGMInline.h"
45
46#include <VBox/sup.h>
47#include <VBox/param.h>
48#include <VBox/err.h>
49#include <VBox/log.h>
50#include <iprt/assert.h>
51#include <iprt/alloc.h>
52#include <iprt/asm.h>
53#ifdef VBOX_STRICT
54# include <iprt/crc.h>
55#endif
56#include <iprt/thread.h>
57#include <iprt/string.h>
58#include <iprt/system.h>
59
60
61/*********************************************************************************************************************************
62* Defined Constants And Macros *
63*********************************************************************************************************************************/
64/** The number of pages to free in one batch. */
65#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
66
67
68
69/*********************************************************************************************************************************
70* Reading and Writing Guest Pysical Memory *
71*********************************************************************************************************************************/
72
73/*
74 * PGMR3PhysReadU8-64
75 * PGMR3PhysWriteU8-64
76 */
77#define PGMPHYSFN_READNAME PGMR3PhysReadU8
78#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
79#define PGMPHYS_DATASIZE 1
80#define PGMPHYS_DATATYPE uint8_t
81#include "PGMPhysRWTmpl.h"
82
83#define PGMPHYSFN_READNAME PGMR3PhysReadU16
84#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
85#define PGMPHYS_DATASIZE 2
86#define PGMPHYS_DATATYPE uint16_t
87#include "PGMPhysRWTmpl.h"
88
89#define PGMPHYSFN_READNAME PGMR3PhysReadU32
90#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
91#define PGMPHYS_DATASIZE 4
92#define PGMPHYS_DATATYPE uint32_t
93#include "PGMPhysRWTmpl.h"
94
95#define PGMPHYSFN_READNAME PGMR3PhysReadU64
96#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
97#define PGMPHYS_DATASIZE 8
98#define PGMPHYS_DATATYPE uint64_t
99#include "PGMPhysRWTmpl.h"
100
101
102/**
103 * EMT worker for PGMR3PhysReadExternal.
104 */
105static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead,
106 PGMACCESSORIGIN enmOrigin)
107{
108 VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead, enmOrigin);
109 AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
110 return VINF_SUCCESS;
111}
112
113
114/**
115 * Read from physical memory, external users.
116 *
117 * @returns VBox status code.
118 * @retval VINF_SUCCESS.
119 *
120 * @param pVM The cross context VM structure.
121 * @param GCPhys Physical address to read from.
122 * @param pvBuf Where to read into.
123 * @param cbRead How many bytes to read.
124 * @param enmOrigin Who is calling.
125 *
126 * @thread Any but EMTs.
127 */
128VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
129{
130 VM_ASSERT_OTHER_THREAD(pVM);
131
132 AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
133 LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
134
135 PGM_LOCK_VOID(pVM);
136
137 /*
138 * Copy loop on ram ranges.
139 */
140 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
141 for (;;)
142 {
143 /* Inside range or not? */
144 if (pRam && GCPhys >= pRam->GCPhys)
145 {
146 /*
147 * Must work our way thru this page by page.
148 */
149 RTGCPHYS off = GCPhys - pRam->GCPhys;
150 while (off < pRam->cb)
151 {
152 unsigned iPage = off >> GUEST_PAGE_SHIFT;
153 PPGMPAGE pPage = &pRam->aPages[iPage];
154
155 /*
156 * If the page has an ALL access handler, we'll have to
157 * delegate the job to EMT.
158 */
159 if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
160 || PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
161 {
162 PGM_UNLOCK(pVM);
163
164 return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 5,
165 pVM, &GCPhys, pvBuf, cbRead, enmOrigin);
166 }
167 Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
168
169 /*
170 * Simple stuff, go ahead.
171 */
172 size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
173 if (cb > cbRead)
174 cb = cbRead;
175 PGMPAGEMAPLOCK PgMpLck;
176 const void *pvSrc;
177 int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
178 if (RT_SUCCESS(rc))
179 {
180 memcpy(pvBuf, pvSrc, cb);
181 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
182 }
183 else
184 {
185 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
186 pRam->GCPhys + off, pPage, rc));
187 memset(pvBuf, 0xff, cb);
188 }
189
190 /* next page */
191 if (cb >= cbRead)
192 {
193 PGM_UNLOCK(pVM);
194 return VINF_SUCCESS;
195 }
196 cbRead -= cb;
197 off += cb;
198 GCPhys += cb;
199 pvBuf = (char *)pvBuf + cb;
200 } /* walk pages in ram range. */
201 }
202 else
203 {
204 LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
205
206 /*
207 * Unassigned address space.
208 */
209 size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
210 if (cb >= cbRead)
211 {
212 memset(pvBuf, 0xff, cbRead);
213 break;
214 }
215 memset(pvBuf, 0xff, cb);
216
217 cbRead -= cb;
218 pvBuf = (char *)pvBuf + cb;
219 GCPhys += cb;
220 }
221
222 /* Advance range if necessary. */
223 while (pRam && GCPhys > pRam->GCPhysLast)
224 pRam = pRam->CTX_SUFF(pNext);
225 } /* Ram range walk */
226
227 PGM_UNLOCK(pVM);
228
229 return VINF_SUCCESS;
230}
231
232
233/**
234 * EMT worker for PGMR3PhysWriteExternal.
235 */
236static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite,
237 PGMACCESSORIGIN enmOrigin)
238{
239 /** @todo VERR_EM_NO_MEMORY */
240 VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite, enmOrigin);
241 AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
242 return VINF_SUCCESS;
243}
244
245
246/**
247 * Write to physical memory, external users.
248 *
249 * @returns VBox status code.
250 * @retval VINF_SUCCESS.
251 * @retval VERR_EM_NO_MEMORY.
252 *
253 * @param pVM The cross context VM structure.
254 * @param GCPhys Physical address to write to.
255 * @param pvBuf What to write.
256 * @param cbWrite How many bytes to write.
257 * @param enmOrigin Who is calling.
258 *
259 * @thread Any but EMTs.
260 */
261VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
262{
263 VM_ASSERT_OTHER_THREAD(pVM);
264
265 AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
266 ("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x enmOrigin=%d\n",
267 GCPhys, cbWrite, enmOrigin));
268 AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
269 LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
270
271 PGM_LOCK_VOID(pVM);
272
273 /*
274 * Copy loop on ram ranges, stop when we hit something difficult.
275 */
276 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
277 for (;;)
278 {
279 /* Inside range or not? */
280 if (pRam && GCPhys >= pRam->GCPhys)
281 {
282 /*
283 * Must work our way thru this page by page.
284 */
285 RTGCPTR off = GCPhys - pRam->GCPhys;
286 while (off < pRam->cb)
287 {
288 RTGCPTR iPage = off >> GUEST_PAGE_SHIFT;
289 PPGMPAGE pPage = &pRam->aPages[iPage];
290
291 /*
292 * Is the page problematic, we have to do the work on the EMT.
293 *
294 * Allocating writable pages and access handlers are
295 * problematic, write monitored pages are simple and can be
296 * dealt with here.
297 */
298 if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
299 || PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
300 || PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
301 {
302 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
303 && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
304 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
305 else
306 {
307 PGM_UNLOCK(pVM);
308
309 return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 5,
310 pVM, &GCPhys, pvBuf, cbWrite, enmOrigin);
311 }
312 }
313 Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
314
315 /*
316 * Simple stuff, go ahead.
317 */
318 size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
319 if (cb > cbWrite)
320 cb = cbWrite;
321 PGMPAGEMAPLOCK PgMpLck;
322 void *pvDst;
323 int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
324 if (RT_SUCCESS(rc))
325 {
326 memcpy(pvDst, pvBuf, cb);
327 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
328 }
329 else
330 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
331 pRam->GCPhys + off, pPage, rc));
332
333 /* next page */
334 if (cb >= cbWrite)
335 {
336 PGM_UNLOCK(pVM);
337 return VINF_SUCCESS;
338 }
339
340 cbWrite -= cb;
341 off += cb;
342 GCPhys += cb;
343 pvBuf = (const char *)pvBuf + cb;
344 } /* walk pages in ram range */
345 }
346 else
347 {
348 /*
349 * Unassigned address space, skip it.
350 */
351 if (!pRam)
352 break;
353 size_t cb = pRam->GCPhys - GCPhys;
354 if (cb >= cbWrite)
355 break;
356 cbWrite -= cb;
357 pvBuf = (const char *)pvBuf + cb;
358 GCPhys += cb;
359 }
360
361 /* Advance range if necessary. */
362 while (pRam && GCPhys > pRam->GCPhysLast)
363 pRam = pRam->CTX_SUFF(pNext);
364 } /* Ram range walk */
365
366 PGM_UNLOCK(pVM);
367 return VINF_SUCCESS;
368}
369
370
371/*********************************************************************************************************************************
372* Mapping Guest Physical Memory *
373*********************************************************************************************************************************/
374
375/**
376 * VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
377 *
378 * @returns see PGMR3PhysGCPhys2CCPtrExternal
379 * @param pVM The cross context VM structure.
380 * @param pGCPhys Pointer to the guest physical address.
381 * @param ppv Where to store the mapping address.
382 * @param pLock Where to store the lock.
383 */
384static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
385{
386 /*
387 * Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
388 * an access handler after it succeeds.
389 */
390 int rc = PGM_LOCK(pVM);
391 AssertRCReturn(rc, rc);
392
393 rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
394 if (RT_SUCCESS(rc))
395 {
396 PPGMPAGEMAPTLBE pTlbe;
397 int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
398 AssertFatalRC(rc2);
399 PPGMPAGE pPage = pTlbe->pPage;
400 if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
401 {
402 PGMPhysReleasePageMappingLock(pVM, pLock);
403 rc = VERR_PGM_PHYS_PAGE_RESERVED;
404 }
405 else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
406#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
407 || pgmPoolIsDirtyPage(pVM, *pGCPhys)
408#endif
409 )
410 {
411 /* We *must* flush any corresponding pgm pool page here, otherwise we'll
412 * not be informed about writes and keep bogus gst->shw mappings around.
413 */
414 pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
415 Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
416 /** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
417 * active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
418 }
419 }
420
421 PGM_UNLOCK(pVM);
422 return rc;
423}
424
425
426/**
427 * Requests the mapping of a guest page into ring-3, external threads.
428 *
429 * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
430 * release it.
431 *
432 * This API will assume your intention is to write to the page, and will
433 * therefore replace shared and zero pages. If you do not intend to modify the
434 * page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
435 *
436 * @returns VBox status code.
437 * @retval VINF_SUCCESS on success.
438 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
439 * backing or if the page has any active access handlers. The caller
440 * must fall back on using PGMR3PhysWriteExternal.
441 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
442 *
443 * @param pVM The cross context VM structure.
444 * @param GCPhys The guest physical address of the page that should be mapped.
445 * @param ppv Where to store the address corresponding to GCPhys.
446 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
447 *
448 * @remark Avoid calling this API from within critical sections (other than the
449 * PGM one) because of the deadlock risk when we have to delegating the
450 * task to an EMT.
451 * @thread Any.
452 */
453VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
454{
455 AssertPtr(ppv);
456 AssertPtr(pLock);
457
458 Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
459
460 int rc = PGM_LOCK(pVM);
461 AssertRCReturn(rc, rc);
462
463 /*
464 * Query the Physical TLB entry for the page (may fail).
465 */
466 PPGMPAGEMAPTLBE pTlbe;
467 rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
468 if (RT_SUCCESS(rc))
469 {
470 PPGMPAGE pPage = pTlbe->pPage;
471 if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
472 rc = VERR_PGM_PHYS_PAGE_RESERVED;
473 else
474 {
475 /*
476 * If the page is shared, the zero page, or being write monitored
477 * it must be converted to an page that's writable if possible.
478 * We can only deal with write monitored pages here, the rest have
479 * to be on an EMT.
480 */
481 if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
482 || PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
483#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
484 || pgmPoolIsDirtyPage(pVM, GCPhys)
485#endif
486 )
487 {
488 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
489 && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
490#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
491 && !pgmPoolIsDirtyPage(pVM, GCPhys) /** @todo we're very likely doing this twice. */
492#endif
493 )
494 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
495 else
496 {
497 PGM_UNLOCK(pVM);
498
499 return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
500 pVM, &GCPhys, ppv, pLock);
501 }
502 }
503
504 /*
505 * Now, just perform the locking and calculate the return address.
506 */
507 PPGMPAGEMAP pMap = pTlbe->pMap;
508 if (pMap)
509 pMap->cRefs++;
510
511 unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
512 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
513 {
514 if (cLocks == 0)
515 pVM->pgm.s.cWriteLockedPages++;
516 PGM_PAGE_INC_WRITE_LOCKS(pPage);
517 }
518 else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
519 {
520 PGM_PAGE_INC_WRITE_LOCKS(pPage);
521 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
522 if (pMap)
523 pMap->cRefs++; /* Extra ref to prevent it from going away. */
524 }
525
526 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
527 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
528 pLock->pvMap = pMap;
529 }
530 }
531
532 PGM_UNLOCK(pVM);
533 return rc;
534}
535
536
537/**
538 * Requests the mapping of a guest page into ring-3, external threads.
539 *
540 * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
541 * release it.
542 *
543 * @returns VBox status code.
544 * @retval VINF_SUCCESS on success.
545 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
546 * backing or if the page as an active ALL access handler. The caller
547 * must fall back on using PGMPhysRead.
548 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
549 *
550 * @param pVM The cross context VM structure.
551 * @param GCPhys The guest physical address of the page that should be mapped.
552 * @param ppv Where to store the address corresponding to GCPhys.
553 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
554 *
555 * @remark Avoid calling this API from within critical sections (other than
556 * the PGM one) because of the deadlock risk.
557 * @thread Any.
558 */
559VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
560{
561 int rc = PGM_LOCK(pVM);
562 AssertRCReturn(rc, rc);
563
564 /*
565 * Query the Physical TLB entry for the page (may fail).
566 */
567 PPGMPAGEMAPTLBE pTlbe;
568 rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
569 if (RT_SUCCESS(rc))
570 {
571 PPGMPAGE pPage = pTlbe->pPage;
572#if 1
573 /* MMIO pages doesn't have any readable backing. */
574 if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
575 rc = VERR_PGM_PHYS_PAGE_RESERVED;
576#else
577 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
578 rc = VERR_PGM_PHYS_PAGE_RESERVED;
579#endif
580 else
581 {
582 /*
583 * Now, just perform the locking and calculate the return address.
584 */
585 PPGMPAGEMAP pMap = pTlbe->pMap;
586 if (pMap)
587 pMap->cRefs++;
588
589 unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
590 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
591 {
592 if (cLocks == 0)
593 pVM->pgm.s.cReadLockedPages++;
594 PGM_PAGE_INC_READ_LOCKS(pPage);
595 }
596 else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
597 {
598 PGM_PAGE_INC_READ_LOCKS(pPage);
599 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
600 if (pMap)
601 pMap->cRefs++; /* Extra ref to prevent it from going away. */
602 }
603
604 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
605 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
606 pLock->pvMap = pMap;
607 }
608 }
609
610 PGM_UNLOCK(pVM);
611 return rc;
612}
613
614
615/**
616 * Requests the mapping of multiple guest page into ring-3, external threads.
617 *
618 * When you're done with the pages, call PGMPhysBulkReleasePageMappingLock()
619 * ASAP to release them.
620 *
621 * This API will assume your intention is to write to the pages, and will
622 * therefore replace shared and zero pages. If you do not intend to modify the
623 * pages, use the PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal() API.
624 *
625 * @returns VBox status code.
626 * @retval VINF_SUCCESS on success.
627 * @retval VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
628 * backing or if any of the pages the page has any active access
629 * handlers. The caller must fall back on using PGMR3PhysWriteExternal.
630 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
631 * an invalid physical address.
632 *
633 * @param pVM The cross context VM structure.
634 * @param cPages Number of pages to lock.
635 * @param paGCPhysPages The guest physical address of the pages that
636 * should be mapped (@a cPages entries).
637 * @param papvPages Where to store the ring-3 mapping addresses
638 * corresponding to @a paGCPhysPages.
639 * @param paLocks Where to store the locking information that
640 * pfnPhysBulkReleasePageMappingLock needs (@a cPages
641 * in length).
642 *
643 * @remark Avoid calling this API from within critical sections (other than the
644 * PGM one) because of the deadlock risk when we have to delegating the
645 * task to an EMT.
646 * @thread Any.
647 */
648VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
649 void **papvPages, PPGMPAGEMAPLOCK paLocks)
650{
651 Assert(cPages > 0);
652 AssertPtr(papvPages);
653 AssertPtr(paLocks);
654
655 Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
656
657 int rc = PGM_LOCK(pVM);
658 AssertRCReturn(rc, rc);
659
660 /*
661 * Lock the pages one by one.
662 * The loop body is similar to PGMR3PhysGCPhys2CCPtrExternal.
663 */
664 int32_t cNextYield = 128;
665 uint32_t iPage;
666 for (iPage = 0; iPage < cPages; iPage++)
667 {
668 if (--cNextYield > 0)
669 { /* likely */ }
670 else
671 {
672 PGM_UNLOCK(pVM);
673 ASMNopPause();
674 PGM_LOCK_VOID(pVM);
675 cNextYield = 128;
676 }
677
678 /*
679 * Query the Physical TLB entry for the page (may fail).
680 */
681 PPGMPAGEMAPTLBE pTlbe;
682 rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
683 if (RT_SUCCESS(rc))
684 { }
685 else
686 break;
687 PPGMPAGE pPage = pTlbe->pPage;
688
689 /*
690 * No MMIO or active access handlers.
691 */
692 if ( !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
693 && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
694 { }
695 else
696 {
697 rc = VERR_PGM_PHYS_PAGE_RESERVED;
698 break;
699 }
700
701 /*
702 * The page must be in the allocated state and not be a dirty pool page.
703 * We can handle converting a write monitored page to an allocated one, but
704 * anything more complicated must be delegated to an EMT.
705 */
706 bool fDelegateToEmt = false;
707 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
708#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
709 fDelegateToEmt = pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]);
710#else
711 fDelegateToEmt = false;
712#endif
713 else if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
714 {
715#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
716 if (!pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]))
717 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, paGCPhysPages[iPage]);
718 else
719 fDelegateToEmt = true;
720#endif
721 }
722 else
723 fDelegateToEmt = true;
724 if (!fDelegateToEmt)
725 { }
726 else
727 {
728 /* We could do this delegation in bulk, but considered too much work vs gain. */
729 PGM_UNLOCK(pVM);
730 rc = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
731 pVM, &paGCPhysPages[iPage], &papvPages[iPage], &paLocks[iPage]);
732 PGM_LOCK_VOID(pVM);
733 if (RT_FAILURE(rc))
734 break;
735 cNextYield = 128;
736 }
737
738 /*
739 * Now, just perform the locking and address calculation.
740 */
741 PPGMPAGEMAP pMap = pTlbe->pMap;
742 if (pMap)
743 pMap->cRefs++;
744
745 unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
746 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
747 {
748 if (cLocks == 0)
749 pVM->pgm.s.cWriteLockedPages++;
750 PGM_PAGE_INC_WRITE_LOCKS(pPage);
751 }
752 else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
753 {
754 PGM_PAGE_INC_WRITE_LOCKS(pPage);
755 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", paGCPhysPages[iPage], pPage));
756 if (pMap)
757 pMap->cRefs++; /* Extra ref to prevent it from going away. */
758 }
759
760 papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(paGCPhysPages[iPage] & GUEST_PAGE_OFFSET_MASK));
761 paLocks[iPage].uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
762 paLocks[iPage].pvMap = pMap;
763 }
764
765 PGM_UNLOCK(pVM);
766
767 /*
768 * On failure we must unlock any pages we managed to get already.
769 */
770 if (RT_FAILURE(rc) && iPage > 0)
771 PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
772
773 return rc;
774}
775
776
777/**
778 * Requests the mapping of multiple guest page into ring-3, for reading only,
779 * external threads.
780 *
781 * When you're done with the pages, call PGMPhysReleasePageMappingLock() ASAP
782 * to release them.
783 *
784 * @returns VBox status code.
785 * @retval VINF_SUCCESS on success.
786 * @retval VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
787 * backing or if any of the pages the page has an active ALL access
788 * handler. The caller must fall back on using PGMR3PhysWriteExternal.
789 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
790 * an invalid physical address.
791 *
792 * @param pVM The cross context VM structure.
793 * @param cPages Number of pages to lock.
794 * @param paGCPhysPages The guest physical address of the pages that
795 * should be mapped (@a cPages entries).
796 * @param papvPages Where to store the ring-3 mapping addresses
797 * corresponding to @a paGCPhysPages.
798 * @param paLocks Where to store the lock information that
799 * pfnPhysReleasePageMappingLock needs (@a cPages
800 * in length).
801 *
802 * @remark Avoid calling this API from within critical sections (other than
803 * the PGM one) because of the deadlock risk.
804 * @thread Any.
805 */
806VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
807 void const **papvPages, PPGMPAGEMAPLOCK paLocks)
808{
809 Assert(cPages > 0);
810 AssertPtr(papvPages);
811 AssertPtr(paLocks);
812
813 Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
814
815 int rc = PGM_LOCK(pVM);
816 AssertRCReturn(rc, rc);
817
818 /*
819 * Lock the pages one by one.
820 * The loop body is similar to PGMR3PhysGCPhys2CCPtrReadOnlyExternal.
821 */
822 int32_t cNextYield = 256;
823 uint32_t iPage;
824 for (iPage = 0; iPage < cPages; iPage++)
825 {
826 if (--cNextYield > 0)
827 { /* likely */ }
828 else
829 {
830 PGM_UNLOCK(pVM);
831 ASMNopPause();
832 PGM_LOCK_VOID(pVM);
833 cNextYield = 256;
834 }
835
836 /*
837 * Query the Physical TLB entry for the page (may fail).
838 */
839 PPGMPAGEMAPTLBE pTlbe;
840 rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
841 if (RT_SUCCESS(rc))
842 { }
843 else
844 break;
845 PPGMPAGE pPage = pTlbe->pPage;
846
847 /*
848 * No MMIO or active all access handlers, everything else can be accessed.
849 */
850 if ( !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
851 && !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
852 { }
853 else
854 {
855 rc = VERR_PGM_PHYS_PAGE_RESERVED;
856 break;
857 }
858
859 /*
860 * Now, just perform the locking and address calculation.
861 */
862 PPGMPAGEMAP pMap = pTlbe->pMap;
863 if (pMap)
864 pMap->cRefs++;
865
866 unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
867 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
868 {
869 if (cLocks == 0)
870 pVM->pgm.s.cReadLockedPages++;
871 PGM_PAGE_INC_READ_LOCKS(pPage);
872 }
873 else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
874 {
875 PGM_PAGE_INC_READ_LOCKS(pPage);
876 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", paGCPhysPages[iPage], pPage));
877 if (pMap)
878 pMap->cRefs++; /* Extra ref to prevent it from going away. */
879 }
880
881 papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(paGCPhysPages[iPage] & GUEST_PAGE_OFFSET_MASK));
882 paLocks[iPage].uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
883 paLocks[iPage].pvMap = pMap;
884 }
885
886 PGM_UNLOCK(pVM);
887
888 /*
889 * On failure we must unlock any pages we managed to get already.
890 */
891 if (RT_FAILURE(rc) && iPage > 0)
892 PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
893
894 return rc;
895}
896
897
898/**
899 * Converts a GC physical address to a HC ring-3 pointer, with some
900 * additional checks.
901 *
902 * @returns VBox status code.
903 * @retval VINF_SUCCESS on success.
904 * @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
905 * access handler of some kind.
906 * @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
907 * accesses or is odd in any way.
908 * @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
909 *
910 * @param pVM The cross context VM structure.
911 * @param GCPhys The GC physical address to convert. Since this is only
912 * used for filling the REM TLB, the A20 mask must be
913 * applied before calling this API.
914 * @param fWritable Whether write access is required.
915 * @param ppv Where to store the pointer corresponding to GCPhys on
916 * success.
917 */
918VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
919{
920 PGM_LOCK_VOID(pVM);
921 PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
922
923 PPGMRAMRANGE pRam;
924 PPGMPAGE pPage;
925 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
926 if (RT_SUCCESS(rc))
927 {
928 if (PGM_PAGE_IS_BALLOONED(pPage))
929 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
930 else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
931 rc = VINF_SUCCESS;
932 else
933 {
934 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
935 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
936 else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
937 {
938 /** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
939 * in -norawr0 mode. */
940 if (fWritable)
941 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
942 }
943 else
944 {
945 /* Temporarily disabled physical handler(s), since the recompiler
946 doesn't get notified when it's reset we'll have to pretend it's
947 operating normally. */
948 if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
949 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
950 else
951 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
952 }
953 }
954 if (RT_SUCCESS(rc))
955 {
956 int rc2;
957
958 /* Make sure what we return is writable. */
959 if (fWritable)
960 switch (PGM_PAGE_GET_STATE(pPage))
961 {
962 case PGM_PAGE_STATE_ALLOCATED:
963 break;
964 case PGM_PAGE_STATE_BALLOONED:
965 AssertFailed();
966 break;
967 case PGM_PAGE_STATE_ZERO:
968 case PGM_PAGE_STATE_SHARED:
969 if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
970 break;
971 RT_FALL_THRU();
972 case PGM_PAGE_STATE_WRITE_MONITORED:
973 rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
974 AssertLogRelRCReturn(rc2, rc2);
975 break;
976 }
977
978 /* Get a ring-3 mapping of the address. */
979 PPGMPAGER3MAPTLBE pTlbe;
980 rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
981 AssertLogRelRCReturn(rc2, rc2);
982 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
983 /** @todo mapping/locking hell; this isn't horribly efficient since
984 * pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
985
986 Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
987 }
988 else
989 Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
990
991 /* else: handler catching all access, no pointer returned. */
992 }
993 else
994 rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
995
996 PGM_UNLOCK(pVM);
997 return rc;
998}
999
1000
1001
1002/*********************************************************************************************************************************
1003* RAM Range Management *
1004*********************************************************************************************************************************/
1005
1006#define MAKE_LEAF(a_pNode) \
1007 do { \
1008 (a_pNode)->pLeftR3 = NIL_RTR3PTR; \
1009 (a_pNode)->pRightR3 = NIL_RTR3PTR; \
1010 (a_pNode)->pLeftR0 = NIL_RTR0PTR; \
1011 (a_pNode)->pRightR0 = NIL_RTR0PTR; \
1012 } while (0)
1013
1014#define INSERT_LEFT(a_pParent, a_pNode) \
1015 do { \
1016 (a_pParent)->pLeftR3 = (a_pNode); \
1017 (a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
1018 } while (0)
1019#define INSERT_RIGHT(a_pParent, a_pNode) \
1020 do { \
1021 (a_pParent)->pRightR3 = (a_pNode); \
1022 (a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
1023 } while (0)
1024
1025
1026/**
1027 * Recursive tree builder.
1028 *
1029 * @param ppRam Pointer to the iterator variable.
1030 * @param iDepth The current depth. Inserts a leaf node if 0.
1031 */
1032static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
1033{
1034 PPGMRAMRANGE pRam;
1035 if (iDepth <= 0)
1036 {
1037 /*
1038 * Leaf node.
1039 */
1040 pRam = *ppRam;
1041 if (pRam)
1042 {
1043 *ppRam = pRam->pNextR3;
1044 MAKE_LEAF(pRam);
1045 }
1046 }
1047 else
1048 {
1049
1050 /*
1051 * Intermediate node.
1052 */
1053 PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
1054
1055 pRam = *ppRam;
1056 if (!pRam)
1057 return pLeft;
1058 *ppRam = pRam->pNextR3;
1059 MAKE_LEAF(pRam);
1060 INSERT_LEFT(pRam, pLeft);
1061
1062 PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
1063 if (pRight)
1064 INSERT_RIGHT(pRam, pRight);
1065 }
1066 return pRam;
1067}
1068
1069
1070/**
1071 * Rebuilds the RAM range search trees.
1072 *
1073 * @param pVM The cross context VM structure.
1074 */
1075static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
1076{
1077
1078 /*
1079 * Create the reasonably balanced tree in a sequential fashion.
1080 * For simplicity (laziness) we use standard recursion here.
1081 */
1082 int iDepth = 0;
1083 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1084 PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
1085 while (pRam)
1086 {
1087 PPGMRAMRANGE pLeft = pRoot;
1088
1089 pRoot = pRam;
1090 pRam = pRam->pNextR3;
1091 MAKE_LEAF(pRoot);
1092 INSERT_LEFT(pRoot, pLeft);
1093
1094 PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
1095 if (pRight)
1096 INSERT_RIGHT(pRoot, pRight);
1097 /** @todo else: rotate the tree. */
1098
1099 iDepth++;
1100 }
1101
1102 pVM->pgm.s.pRamRangeTreeR3 = pRoot;
1103 pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
1104
1105#ifdef VBOX_STRICT
1106 /*
1107 * Verify that the above code works.
1108 */
1109 unsigned cRanges = 0;
1110 for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1111 cRanges++;
1112 Assert(cRanges > 0);
1113
1114 unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
1115 if ((1U << cMaxDepth) < cRanges)
1116 cMaxDepth++;
1117
1118 for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1119 {
1120 unsigned cDepth = 0;
1121 PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
1122 for (;;)
1123 {
1124 if (pRam == pRam2)
1125 break;
1126 Assert(pRam2);
1127 if (pRam->GCPhys < pRam2->GCPhys)
1128 pRam2 = pRam2->pLeftR3;
1129 else
1130 pRam2 = pRam2->pRightR3;
1131 }
1132 AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
1133 }
1134#endif /* VBOX_STRICT */
1135}
1136
1137#undef MAKE_LEAF
1138#undef INSERT_LEFT
1139#undef INSERT_RIGHT
1140
1141/**
1142 * Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
1143 *
1144 * Called when anything was relocated.
1145 *
1146 * @param pVM The cross context VM structure.
1147 */
1148void pgmR3PhysRelinkRamRanges(PVM pVM)
1149{
1150 PPGMRAMRANGE pCur;
1151
1152#ifdef VBOX_STRICT
1153 for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
1154 {
1155 Assert((pCur->GCPhys & GUEST_PAGE_OFFSET_MASK) == 0);
1156 Assert((pCur->GCPhysLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK);
1157 Assert((pCur->cb & GUEST_PAGE_OFFSET_MASK) == 0);
1158 Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
1159 for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
1160 Assert( pCur2 == pCur
1161 || strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
1162 }
1163#endif
1164
1165 pCur = pVM->pgm.s.pRamRangesXR3;
1166 if (pCur)
1167 {
1168 pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
1169
1170 for (; pCur->pNextR3; pCur = pCur->pNextR3)
1171 pCur->pNextR0 = pCur->pNextR3->pSelfR0;
1172
1173 Assert(pCur->pNextR0 == NIL_RTR0PTR);
1174 }
1175 else
1176 {
1177 Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
1178 }
1179 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
1180
1181 pgmR3PhysRebuildRamRangeSearchTrees(pVM);
1182}
1183
1184
1185/**
1186 * Links a new RAM range into the list.
1187 *
1188 * @param pVM The cross context VM structure.
1189 * @param pNew Pointer to the new list entry.
1190 * @param pPrev Pointer to the previous list entry. If NULL, insert as head.
1191 */
1192static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
1193{
1194 AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
1195
1196 PGM_LOCK_VOID(pVM);
1197
1198 PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
1199 pNew->pNextR3 = pRam;
1200 pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
1201
1202 if (pPrev)
1203 {
1204 pPrev->pNextR3 = pNew;
1205 pPrev->pNextR0 = pNew->pSelfR0;
1206 }
1207 else
1208 {
1209 pVM->pgm.s.pRamRangesXR3 = pNew;
1210 pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
1211 }
1212 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
1213
1214 pgmR3PhysRebuildRamRangeSearchTrees(pVM);
1215 PGM_UNLOCK(pVM);
1216}
1217
1218
1219/**
1220 * Unlink an existing RAM range from the list.
1221 *
1222 * @param pVM The cross context VM structure.
1223 * @param pRam Pointer to the new list entry.
1224 * @param pPrev Pointer to the previous list entry. If NULL, insert as head.
1225 */
1226static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
1227{
1228 Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
1229
1230 PGM_LOCK_VOID(pVM);
1231
1232 PPGMRAMRANGE pNext = pRam->pNextR3;
1233 if (pPrev)
1234 {
1235 pPrev->pNextR3 = pNext;
1236 pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
1237 }
1238 else
1239 {
1240 Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
1241 pVM->pgm.s.pRamRangesXR3 = pNext;
1242 pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
1243 }
1244 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
1245
1246 pgmR3PhysRebuildRamRangeSearchTrees(pVM);
1247 PGM_UNLOCK(pVM);
1248}
1249
1250
1251/**
1252 * Unlink an existing RAM range from the list.
1253 *
1254 * @param pVM The cross context VM structure.
1255 * @param pRam Pointer to the new list entry.
1256 */
1257static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
1258{
1259 PGM_LOCK_VOID(pVM);
1260
1261 /* find prev. */
1262 PPGMRAMRANGE pPrev = NULL;
1263 PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
1264 while (pCur != pRam)
1265 {
1266 pPrev = pCur;
1267 pCur = pCur->pNextR3;
1268 }
1269 AssertFatal(pCur);
1270
1271 pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
1272 PGM_UNLOCK(pVM);
1273}
1274
1275
1276/**
1277 * Gets the number of ram ranges.
1278 *
1279 * @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1280 * @param pVM The cross context VM structure.
1281 */
1282VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1283{
1284 VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1285
1286 PGM_LOCK_VOID(pVM);
1287 uint32_t cRamRanges = 0;
1288 for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1289 cRamRanges++;
1290 PGM_UNLOCK(pVM);
1291 return cRamRanges;
1292}
1293
1294
1295/**
1296 * Get information about a range.
1297 *
1298 * @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1299 * @param pVM The cross context VM structure.
1300 * @param iRange The ordinal of the range.
1301 * @param pGCPhysStart Where to return the start of the range. Optional.
1302 * @param pGCPhysLast Where to return the address of the last byte in the
1303 * range. Optional.
1304 * @param ppszDesc Where to return the range description. Optional.
1305 * @param pfIsMmio Where to indicate that this is a pure MMIO range.
1306 * Optional.
1307 */
1308VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1309 const char **ppszDesc, bool *pfIsMmio)
1310{
1311 VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1312
1313 PGM_LOCK_VOID(pVM);
1314 uint32_t iCurRange = 0;
1315 for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1316 if (iCurRange == iRange)
1317 {
1318 if (pGCPhysStart)
1319 *pGCPhysStart = pCur->GCPhys;
1320 if (pGCPhysLast)
1321 *pGCPhysLast = pCur->GCPhysLast;
1322 if (ppszDesc)
1323 *ppszDesc = pCur->pszDesc;
1324 if (pfIsMmio)
1325 *pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1326
1327 PGM_UNLOCK(pVM);
1328 return VINF_SUCCESS;
1329 }
1330 PGM_UNLOCK(pVM);
1331 return VERR_OUT_OF_RANGE;
1332}
1333
1334
1335/*********************************************************************************************************************************
1336* RAM *
1337*********************************************************************************************************************************/
1338
1339/**
1340 * Frees the specified RAM page and replaces it with the ZERO page.
1341 *
1342 * This is used by ballooning, remapping MMIO2, RAM reset and state loading.
1343 *
1344 * @param pVM The cross context VM structure.
1345 * @param pReq Pointer to the request. This is NULL when doing a
1346 * bulk free in NEM memory mode.
1347 * @param pcPendingPages Where the number of pages waiting to be freed are
1348 * kept. This will normally be incremented. This is
1349 * NULL when doing a bulk free in NEM memory mode.
1350 * @param pPage Pointer to the page structure.
1351 * @param GCPhys The guest physical address of the page, if applicable.
1352 * @param enmNewType New page type for NEM notification, since several
1353 * callers will change the type upon successful return.
1354 *
1355 * @remarks The caller must own the PGM lock.
1356 */
1357int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys,
1358 PGMPAGETYPE enmNewType)
1359{
1360 /*
1361 * Assert sanity.
1362 */
1363 PGM_LOCK_ASSERT_OWNER(pVM);
1364 if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
1365 && PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
1366 {
1367 AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
1368 return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
1369 }
1370
1371 /** @todo What about ballooning of large pages??! */
1372 Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
1373 && PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
1374
1375 if ( PGM_PAGE_IS_ZERO(pPage)
1376 || PGM_PAGE_IS_BALLOONED(pPage))
1377 return VINF_SUCCESS;
1378
1379 const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
1380 Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
1381 if (RT_UNLIKELY(!PGM_IS_IN_NEM_MODE(pVM)
1382 ? idPage == NIL_GMM_PAGEID
1383 || idPage > GMM_PAGEID_LAST
1384 || PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID
1385 : idPage != NIL_GMM_PAGEID))
1386 {
1387 AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
1388 return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
1389 }
1390#ifdef VBOX_WITH_NATIVE_NEM
1391 const RTHCPHYS HCPhysPrev = PGM_PAGE_GET_HCPHYS(pPage);
1392#endif
1393
1394 /* update page count stats. */
1395 if (PGM_PAGE_IS_SHARED(pPage))
1396 pVM->pgm.s.cSharedPages--;
1397 else
1398 pVM->pgm.s.cPrivatePages--;
1399 pVM->pgm.s.cZeroPages++;
1400
1401 /* Deal with write monitored pages. */
1402 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
1403 {
1404 PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
1405 pVM->pgm.s.cWrittenToPages++;
1406 }
1407
1408 /*
1409 * pPage = ZERO page.
1410 */
1411 PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
1412 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1413 PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
1414 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
1415 PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
1416 PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
1417
1418 /* Flush physical page map TLB entry. */
1419 pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
1420 IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID); /// @todo move to the perform step.
1421
1422#ifdef VBOX_WITH_PGM_NEM_MODE
1423 /*
1424 * Skip the rest if we're doing a bulk free in NEM memory mode.
1425 */
1426 if (!pReq)
1427 return VINF_SUCCESS;
1428 AssertLogRelReturn(!pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
1429#endif
1430
1431#ifdef VBOX_WITH_NATIVE_NEM
1432 /* Notify NEM. */
1433 /** @todo Remove this one? */
1434 if (VM_IS_NEM_ENABLED(pVM))
1435 {
1436 uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
1437 NEMHCNotifyPhysPageChanged(pVM, GCPhys, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg, pVM->pgm.s.abZeroPg,
1438 pgmPhysPageCalcNemProtection(pPage, enmNewType), enmNewType, &u2State);
1439 PGM_PAGE_SET_NEM_STATE(pPage, u2State);
1440 }
1441#else
1442 RT_NOREF(enmNewType);
1443#endif
1444
1445 /*
1446 * Make sure it's not in the handy page array.
1447 */
1448 for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
1449 {
1450 if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
1451 {
1452 pVM->pgm.s.aHandyPages[i].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS;
1453 pVM->pgm.s.aHandyPages[i].fZeroed = false;
1454 pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
1455 break;
1456 }
1457 if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
1458 {
1459 pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
1460 break;
1461 }
1462 }
1463
1464 /*
1465 * Push it onto the page array.
1466 */
1467 uint32_t iPage = *pcPendingPages;
1468 Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
1469 *pcPendingPages += 1;
1470
1471 pReq->aPages[iPage].idPage = idPage;
1472
1473 if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
1474 return VINF_SUCCESS;
1475
1476 /*
1477 * Flush the pages.
1478 */
1479 int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
1480 if (RT_SUCCESS(rc))
1481 {
1482 GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1483 *pcPendingPages = 0;
1484 }
1485 return rc;
1486}
1487
1488
1489/**
1490 * Frees a range of pages, replacing them with ZERO pages of the specified type.
1491 *
1492 * @returns VBox status code.
1493 * @param pVM The cross context VM structure.
1494 * @param pRam The RAM range in which the pages resides.
1495 * @param GCPhys The address of the first page.
1496 * @param GCPhysLast The address of the last page.
1497 * @param pvMmio2 Pointer to the ring-3 mapping of any MMIO2 memory that
1498 * will replace the pages we're freeing up.
1499 */
1500static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, void *pvMmio2)
1501{
1502 PGM_LOCK_ASSERT_OWNER(pVM);
1503
1504#ifdef VBOX_WITH_PGM_NEM_MODE
1505 /*
1506 * In simplified memory mode we don't actually free the memory,
1507 * we just unmap it and let NEM do any unlocking of it.
1508 */
1509 if (pVM->pgm.s.fNemMode)
1510 {
1511 Assert(VM_IS_NEM_ENABLED(pVM) || VM_IS_EXEC_ENGINE_IEM(pVM));
1512 uint8_t u2State = 0; /* (We don't support UINT8_MAX here.) */
1513 if (VM_IS_NEM_ENABLED(pVM))
1514 {
1515 uint32_t const fNemNotify = (pvMmio2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0) | NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE;
1516 int rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify,
1517 pRam->pvR3 ? (uint8_t *)pRam->pvR3 + GCPhys - pRam->GCPhys : NULL,
1518 pvMmio2, &u2State, NULL /*puNemRange*/);
1519 AssertLogRelRCReturn(rc, rc);
1520 }
1521
1522 /* Iterate the pages. */
1523 PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
1524 uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> GUEST_PAGE_SHIFT) + 1;
1525 while (cPagesLeft-- > 0)
1526 {
1527 int rc = pgmPhysFreePage(pVM, NULL, NULL, pPageDst, GCPhys, PGMPAGETYPE_MMIO);
1528 AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
1529
1530 PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO);
1531 PGM_PAGE_SET_NEM_STATE(pPageDst, u2State);
1532
1533 GCPhys += GUEST_PAGE_SIZE;
1534 pPageDst++;
1535 }
1536 return VINF_SUCCESS;
1537 }
1538#else /* !VBOX_WITH_PGM_NEM_MODE */
1539 RT_NOREF(pvMmio2);
1540#endif /* !VBOX_WITH_PGM_NEM_MODE */
1541
1542 /*
1543 * Regular mode.
1544 */
1545 /* Prepare. */
1546 uint32_t cPendingPages = 0;
1547 PGMMFREEPAGESREQ pReq;
1548 int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1549 AssertLogRelRCReturn(rc, rc);
1550
1551#ifdef VBOX_WITH_NATIVE_NEM
1552 /* Tell NEM up-front. */
1553 uint8_t u2State = UINT8_MAX;
1554 if (VM_IS_NEM_ENABLED(pVM))
1555 {
1556 uint32_t const fNemNotify = (pvMmio2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0) | NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE;
1557 rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify, NULL, pvMmio2,
1558 &u2State, NULL /*puNemRange*/);
1559 AssertLogRelRCReturnStmt(rc, GMMR3FreePagesCleanup(pReq), rc);
1560 }
1561#endif
1562
1563 /* Iterate the pages. */
1564 PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
1565 uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> GUEST_PAGE_SHIFT) + 1;
1566 while (cPagesLeft-- > 0)
1567 {
1568 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys, PGMPAGETYPE_MMIO);
1569 AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
1570
1571 PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO);
1572#ifdef VBOX_WITH_NATIVE_NEM
1573 if (u2State != UINT8_MAX)
1574 PGM_PAGE_SET_NEM_STATE(pPageDst, u2State);
1575#endif
1576
1577 GCPhys += GUEST_PAGE_SIZE;
1578 pPageDst++;
1579 }
1580
1581 /* Finish pending and cleanup. */
1582 if (cPendingPages)
1583 {
1584 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1585 AssertLogRelRCReturn(rc, rc);
1586 }
1587 GMMR3FreePagesCleanup(pReq);
1588
1589 return rc;
1590}
1591
1592
1593/**
1594 * PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1595 *
1596 * In NEM mode, this will allocate the pages backing the RAM range and this may
1597 * fail. NEM registration may also fail. (In regular HM mode it won't fail.)
1598 *
1599 * @returns VBox status code.
1600 * @param pVM The cross context VM structure.
1601 * @param pNew The new RAM range.
1602 * @param GCPhys The address of the RAM range.
1603 * @param GCPhysLast The last address of the RAM range.
1604 * @param R0PtrNew Ditto for R0.
1605 * @param fFlags PGM_RAM_RANGE_FLAGS_FLOATING or zero.
1606 * @param pszDesc The description.
1607 * @param pPrev The previous RAM range (for linking).
1608 */
1609static int pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1610 RTR0PTR R0PtrNew, uint32_t fFlags, const char *pszDesc, PPGMRAMRANGE pPrev)
1611{
1612 /*
1613 * Initialize the range.
1614 */
1615 pNew->pSelfR0 = R0PtrNew;
1616 pNew->GCPhys = GCPhys;
1617 pNew->GCPhysLast = GCPhysLast;
1618 pNew->cb = GCPhysLast - GCPhys + 1;
1619 pNew->pszDesc = pszDesc;
1620 pNew->fFlags = fFlags;
1621 pNew->uNemRange = UINT32_MAX;
1622 pNew->pvR3 = NULL;
1623 pNew->paLSPages = NULL;
1624
1625 uint32_t const cPages = pNew->cb >> GUEST_PAGE_SHIFT;
1626#ifdef VBOX_WITH_PGM_NEM_MODE
1627 if (!pVM->pgm.s.fNemMode)
1628#endif
1629 {
1630 RTGCPHYS iPage = cPages;
1631 while (iPage-- > 0)
1632 PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1633
1634 /* Update the page count stats. */
1635 pVM->pgm.s.cZeroPages += cPages;
1636 pVM->pgm.s.cAllPages += cPages;
1637 }
1638#ifdef VBOX_WITH_PGM_NEM_MODE
1639 else
1640 {
1641 int rc = SUPR3PageAlloc(RT_ALIGN_Z(pNew->cb, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT,
1642 pVM->pgm.s.fUseLargePages ? SUP_PAGE_ALLOC_F_LARGE_PAGES : 0, &pNew->pvR3);
1643 if (RT_FAILURE(rc))
1644 return rc;
1645
1646 RTGCPHYS iPage = cPages;
1647 while (iPage-- > 0)
1648 PGM_PAGE_INIT(&pNew->aPages[iPage], UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
1649 PGMPAGETYPE_RAM, PGM_PAGE_STATE_ALLOCATED);
1650
1651 /* Update the page count stats. */
1652 pVM->pgm.s.cPrivatePages += cPages;
1653 pVM->pgm.s.cAllPages += cPages;
1654 }
1655#endif
1656
1657 /*
1658 * Link it.
1659 */
1660 pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1661
1662#ifdef VBOX_WITH_NATIVE_NEM
1663 /*
1664 * Notify NEM now that it has been linked.
1665 */
1666 if (VM_IS_NEM_ENABLED(pVM))
1667 {
1668 uint8_t u2State = UINT8_MAX;
1669 int rc = NEMR3NotifyPhysRamRegister(pVM, GCPhys, pNew->cb, pNew->pvR3, &u2State, &pNew->uNemRange);
1670 if (RT_SUCCESS(rc))
1671 {
1672 if (u2State != UINT8_MAX)
1673 pgmPhysSetNemStateForPages(&pNew->aPages[0], cPages, u2State);
1674 }
1675 else
1676 pgmR3PhysUnlinkRamRange2(pVM, pNew, pPrev);
1677 return rc;
1678 }
1679#endif
1680 return VINF_SUCCESS;
1681}
1682
1683
1684/**
1685 * PGMR3PhysRegisterRam worker that registers a high chunk.
1686 *
1687 * @returns VBox status code.
1688 * @param pVM The cross context VM structure.
1689 * @param GCPhys The address of the RAM.
1690 * @param cRamPages The number of RAM pages to register.
1691 * @param iChunk The chunk number.
1692 * @param pszDesc The RAM range description.
1693 * @param ppPrev Previous RAM range pointer. In/Out.
1694 */
1695static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages, uint32_t iChunk,
1696 const char *pszDesc, PPGMRAMRANGE *ppPrev)
1697{
1698 const char *pszDescChunk = iChunk == 0
1699 ? pszDesc
1700 : MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1701 AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1702
1703 /*
1704 * Allocate memory for the new chunk.
1705 */
1706 size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cRamPages]), HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT;
1707 PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1708 AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1709 RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1710 void *pvChunk = NULL;
1711 int rc = SUPR3PageAllocEx(cChunkPages, 0 /*fFlags*/, &pvChunk, &R0PtrChunk, paChunkPages);
1712 if (RT_SUCCESS(rc))
1713 {
1714 Assert(R0PtrChunk != NIL_RTR0PTR || PGM_IS_IN_NEM_MODE(pVM));
1715 memset(pvChunk, 0, cChunkPages << HOST_PAGE_SHIFT);
1716
1717 PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1718
1719 /*
1720 * Ok, init and link the range.
1721 */
1722 rc = pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << GUEST_PAGE_SHIFT) - 1,
1723 R0PtrChunk, PGM_RAM_RANGE_FLAGS_FLOATING, pszDescChunk, *ppPrev);
1724 if (RT_SUCCESS(rc))
1725 *ppPrev = pNew;
1726
1727 if (RT_FAILURE(rc))
1728 SUPR3PageFreeEx(pvChunk, cChunkPages);
1729 }
1730
1731 RTMemTmpFree(paChunkPages);
1732 return rc;
1733}
1734
1735
1736/**
1737 * Sets up a range RAM.
1738 *
1739 * This will check for conflicting registrations, make a resource
1740 * reservation for the memory (with GMM), and setup the per-page
1741 * tracking structures (PGMPAGE).
1742 *
1743 * @returns VBox status code.
1744 * @param pVM The cross context VM structure.
1745 * @param GCPhys The physical address of the RAM.
1746 * @param cb The size of the RAM.
1747 * @param pszDesc The description - not copied, so, don't free or change it.
1748 */
1749VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1750{
1751 /*
1752 * Validate input.
1753 */
1754 Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1755 AssertReturn(RT_ALIGN_T(GCPhys, GUEST_PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1756 AssertReturn(RT_ALIGN_T(cb, GUEST_PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1757 AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1758 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1759 AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1760 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1761 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1762
1763 PGM_LOCK_VOID(pVM);
1764
1765 /*
1766 * Find range location and check for conflicts.
1767 */
1768 PPGMRAMRANGE pPrev = NULL;
1769 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1770 while (pRam && GCPhysLast >= pRam->GCPhys)
1771 {
1772 AssertLogRelMsgReturnStmt( GCPhysLast < pRam->GCPhys
1773 || GCPhys > pRam->GCPhysLast,
1774 ("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1775 GCPhys, GCPhysLast, pszDesc, pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1776 PGM_UNLOCK(pVM), VERR_PGM_RAM_CONFLICT);
1777
1778 /* next */
1779 pPrev = pRam;
1780 pRam = pRam->pNextR3;
1781 }
1782
1783 /*
1784 * Register it with GMM (the API bitches).
1785 */
1786 const RTGCPHYS cPages = cb >> GUEST_PAGE_SHIFT;
1787 int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1788 if (RT_FAILURE(rc))
1789 {
1790 PGM_UNLOCK(pVM);
1791 return rc;
1792 }
1793
1794 if ( GCPhys >= _4G
1795 && cPages > 256)
1796 {
1797 /*
1798 * The PGMRAMRANGE structures for the high memory can get very big.
1799 * There used to be some limitations on SUPR3PageAllocEx allocation
1800 * sizes, so traditionally we limited this to 16MB chunks. These days
1801 * we do ~64 MB chunks each covering 16GB of guest RAM, making sure
1802 * each range is a multiple of 1GB to enable eager hosts to use 1GB
1803 * pages in NEM mode.
1804 *
1805 * See also pgmR3PhysMmio2CalcChunkCount.
1806 */
1807 uint32_t const cPagesPerChunk = _4M;
1808 Assert(RT_ALIGN_32(cPagesPerChunk, X86_PD_PAE_SHIFT - X86_PAGE_SHIFT)); /* NEM large page requirement: 1GB pages. */
1809
1810 RTGCPHYS cPagesLeft = cPages;
1811 RTGCPHYS GCPhysChunk = GCPhys;
1812 uint32_t iChunk = 0;
1813 while (cPagesLeft > 0)
1814 {
1815 uint32_t cPagesInChunk = cPagesLeft;
1816 if (cPagesInChunk > cPagesPerChunk)
1817 cPagesInChunk = cPagesPerChunk;
1818
1819 rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, iChunk, pszDesc, &pPrev);
1820 AssertRCReturn(rc, rc);
1821
1822 /* advance */
1823 GCPhysChunk += (RTGCPHYS)cPagesInChunk << GUEST_PAGE_SHIFT;
1824 cPagesLeft -= cPagesInChunk;
1825 iChunk++;
1826 }
1827 }
1828 else
1829 {
1830 /*
1831 * Allocate, initialize and link the new RAM range.
1832 */
1833 const size_t cbRamRange = RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]);
1834 PPGMRAMRANGE pNew = NULL;
1835 RTR0PTR pNewR0 = NIL_RTR0PTR;
1836 rc = SUPR3PageAllocEx(RT_ALIGN_Z(cbRamRange, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT, 0 /*fFlags*/,
1837 (void **)&pNew, &pNewR0, NULL /*paPages*/);
1838 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc cbRamRange=%zu\n", rc, cbRamRange), rc);
1839
1840 rc = pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, pNewR0, 0 /*fFlags*/, pszDesc, pPrev);
1841 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc cbRamRange=%zu\n", rc, cbRamRange), rc);
1842 }
1843 pgmPhysInvalidatePageMapTLB(pVM);
1844
1845 PGM_UNLOCK(pVM);
1846 return rc;
1847}
1848
1849
1850/**
1851 * Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1852 *
1853 * We do this late in the init process so that all the ROM and MMIO ranges have
1854 * been registered already and we don't go wasting memory on them.
1855 *
1856 * @returns VBox status code.
1857 *
1858 * @param pVM The cross context VM structure.
1859 */
1860int pgmR3PhysRamPreAllocate(PVM pVM)
1861{
1862 Assert(pVM->pgm.s.fRamPreAlloc);
1863 Log(("pgmR3PhysRamPreAllocate: enter\n"));
1864#ifdef VBOX_WITH_PGM_NEM_MODE
1865 AssertLogRelReturn(!pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
1866#endif
1867
1868 /*
1869 * Walk the RAM ranges and allocate all RAM pages, halt at
1870 * the first allocation error.
1871 */
1872 uint64_t cPages = 0;
1873 uint64_t NanoTS = RTTimeNanoTS();
1874 PGM_LOCK_VOID(pVM);
1875 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1876 {
1877 PPGMPAGE pPage = &pRam->aPages[0];
1878 RTGCPHYS GCPhys = pRam->GCPhys;
1879 uint32_t cLeft = pRam->cb >> GUEST_PAGE_SHIFT;
1880 while (cLeft-- > 0)
1881 {
1882 if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1883 {
1884 switch (PGM_PAGE_GET_STATE(pPage))
1885 {
1886 case PGM_PAGE_STATE_ZERO:
1887 {
1888 int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1889 if (RT_FAILURE(rc))
1890 {
1891 LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1892 PGM_UNLOCK(pVM);
1893 return rc;
1894 }
1895 cPages++;
1896 break;
1897 }
1898
1899 case PGM_PAGE_STATE_BALLOONED:
1900 case PGM_PAGE_STATE_ALLOCATED:
1901 case PGM_PAGE_STATE_WRITE_MONITORED:
1902 case PGM_PAGE_STATE_SHARED:
1903 /* nothing to do here. */
1904 break;
1905 }
1906 }
1907
1908 /* next */
1909 pPage++;
1910 GCPhys += GUEST_PAGE_SIZE;
1911 }
1912 }
1913 PGM_UNLOCK(pVM);
1914 NanoTS = RTTimeNanoTS() - NanoTS;
1915
1916 LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1917 Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1918 return VINF_SUCCESS;
1919}
1920
1921
1922/**
1923 * Checks shared page checksums.
1924 *
1925 * @param pVM The cross context VM structure.
1926 */
1927void pgmR3PhysAssertSharedPageChecksums(PVM pVM)
1928{
1929#ifdef VBOX_STRICT
1930 PGM_LOCK_VOID(pVM);
1931
1932 if (pVM->pgm.s.cSharedPages > 0)
1933 {
1934 /*
1935 * Walk the ram ranges.
1936 */
1937 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1938 {
1939 uint32_t iPage = pRam->cb >> GUEST_PAGE_SHIFT;
1940 AssertMsg(((RTGCPHYS)iPage << GUEST_PAGE_SHIFT) == pRam->cb,
1941 ("%RGp %RGp\n", (RTGCPHYS)iPage << GUEST_PAGE_SHIFT, pRam->cb));
1942
1943 while (iPage-- > 0)
1944 {
1945 PPGMPAGE pPage = &pRam->aPages[iPage];
1946 if (PGM_PAGE_IS_SHARED(pPage))
1947 {
1948 uint32_t u32Checksum = pPage->s.u2Unused0/* | ((uint32_t)pPage->s.u2Unused1 << 8)*/;
1949 if (!u32Checksum)
1950 {
1951 RTGCPHYS GCPhysPage = pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT);
1952 void const *pvPage;
1953 int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhysPage, &pvPage);
1954 if (RT_SUCCESS(rc))
1955 {
1956 uint32_t u32Checksum2 = RTCrc32(pvPage, GUEST_PAGE_SIZE);
1957# if 0
1958 AssertMsg((u32Checksum2 & /*UINT32_C(0x00000303)*/ 0x3) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
1959# else
1960 if ((u32Checksum2 & /*UINT32_C(0x00000303)*/ 0x3) == u32Checksum)
1961 LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1962 else
1963 AssertMsgFailed(("shpg %#x @ %RGp %#x\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1964# endif
1965 }
1966 else
1967 AssertRC(rc);
1968 }
1969 }
1970
1971 } /* for each page */
1972
1973 } /* for each ram range */
1974 }
1975
1976 PGM_UNLOCK(pVM);
1977#endif /* VBOX_STRICT */
1978 NOREF(pVM);
1979}
1980
1981
1982/**
1983 * Resets the physical memory state.
1984 *
1985 * ASSUMES that the caller owns the PGM lock.
1986 *
1987 * @returns VBox status code.
1988 * @param pVM The cross context VM structure.
1989 */
1990int pgmR3PhysRamReset(PVM pVM)
1991{
1992 PGM_LOCK_ASSERT_OWNER(pVM);
1993
1994 /* Reset the memory balloon. */
1995 int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1996 AssertRC(rc);
1997
1998#ifdef VBOX_WITH_PAGE_SHARING
1999 /* Clear all registered shared modules. */
2000 pgmR3PhysAssertSharedPageChecksums(pVM);
2001 rc = GMMR3ResetSharedModules(pVM);
2002 AssertRC(rc);
2003#endif
2004 /* Reset counters. */
2005 pVM->pgm.s.cReusedSharedPages = 0;
2006 pVM->pgm.s.cBalloonedPages = 0;
2007
2008 return VINF_SUCCESS;
2009}
2010
2011
2012/**
2013 * Resets (zeros) the RAM after all devices and components have been reset.
2014 *
2015 * ASSUMES that the caller owns the PGM lock.
2016 *
2017 * @returns VBox status code.
2018 * @param pVM The cross context VM structure.
2019 */
2020int pgmR3PhysRamZeroAll(PVM pVM)
2021{
2022 PGM_LOCK_ASSERT_OWNER(pVM);
2023
2024 /*
2025 * We batch up pages that should be freed instead of calling GMM for
2026 * each and every one of them.
2027 */
2028 uint32_t cPendingPages = 0;
2029 PGMMFREEPAGESREQ pReq;
2030 int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2031 AssertLogRelRCReturn(rc, rc);
2032
2033 /*
2034 * Walk the ram ranges.
2035 */
2036 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2037 {
2038 uint32_t iPage = pRam->cb >> GUEST_PAGE_SHIFT;
2039 AssertMsg(((RTGCPHYS)iPage << GUEST_PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << GUEST_PAGE_SHIFT, pRam->cb));
2040
2041 if ( !pVM->pgm.s.fRamPreAlloc
2042#ifdef VBOX_WITH_PGM_NEM_MODE
2043 && !pVM->pgm.s.fNemMode
2044#endif
2045 && pVM->pgm.s.fZeroRamPagesOnReset)
2046 {
2047 /* Replace all RAM pages by ZERO pages. */
2048 while (iPage-- > 0)
2049 {
2050 PPGMPAGE pPage = &pRam->aPages[iPage];
2051 switch (PGM_PAGE_GET_TYPE(pPage))
2052 {
2053 case PGMPAGETYPE_RAM:
2054 /* Do not replace pages part of a 2 MB continuous range
2055 with zero pages, but zero them instead. */
2056 if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
2057 || PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
2058 {
2059 void *pvPage;
2060 rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), &pvPage);
2061 AssertLogRelRCReturn(rc, rc);
2062 RT_BZERO(pvPage, GUEST_PAGE_SIZE);
2063 }
2064 else if (PGM_PAGE_IS_BALLOONED(pPage))
2065 {
2066 /* Turn into a zero page; the balloon status is lost when the VM reboots. */
2067 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2068 }
2069 else if (!PGM_PAGE_IS_ZERO(pPage))
2070 {
2071 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage,
2072 pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), PGMPAGETYPE_RAM);
2073 AssertLogRelRCReturn(rc, rc);
2074 }
2075 break;
2076
2077 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2078 case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2079 pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT),
2080 pRam, true /*fDoAccounting*/);
2081 break;
2082
2083 case PGMPAGETYPE_MMIO2:
2084 case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2085 case PGMPAGETYPE_ROM:
2086 case PGMPAGETYPE_MMIO:
2087 break;
2088 default:
2089 AssertFailed();
2090 }
2091 } /* for each page */
2092 }
2093 else
2094 {
2095 /* Zero the memory. */
2096 while (iPage-- > 0)
2097 {
2098 PPGMPAGE pPage = &pRam->aPages[iPage];
2099 switch (PGM_PAGE_GET_TYPE(pPage))
2100 {
2101 case PGMPAGETYPE_RAM:
2102 switch (PGM_PAGE_GET_STATE(pPage))
2103 {
2104 case PGM_PAGE_STATE_ZERO:
2105 break;
2106
2107 case PGM_PAGE_STATE_BALLOONED:
2108 /* Turn into a zero page; the balloon status is lost when the VM reboots. */
2109 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2110 break;
2111
2112 case PGM_PAGE_STATE_SHARED:
2113 case PGM_PAGE_STATE_WRITE_MONITORED:
2114 rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT));
2115 AssertLogRelRCReturn(rc, rc);
2116 RT_FALL_THRU();
2117
2118 case PGM_PAGE_STATE_ALLOCATED:
2119 if (pVM->pgm.s.fZeroRamPagesOnReset)
2120 {
2121 void *pvPage;
2122 rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), &pvPage);
2123 AssertLogRelRCReturn(rc, rc);
2124 RT_BZERO(pvPage, GUEST_PAGE_SIZE);
2125 }
2126 break;
2127 }
2128 break;
2129
2130 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2131 case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2132 pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT),
2133 pRam, true /*fDoAccounting*/);
2134 break;
2135
2136 case PGMPAGETYPE_MMIO2:
2137 case PGMPAGETYPE_ROM_SHADOW:
2138 case PGMPAGETYPE_ROM:
2139 case PGMPAGETYPE_MMIO:
2140 break;
2141 default:
2142 AssertFailed();
2143
2144 }
2145 } /* for each page */
2146 }
2147
2148 }
2149
2150 /*
2151 * Finish off any pages pending freeing.
2152 */
2153 if (cPendingPages)
2154 {
2155 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2156 AssertLogRelRCReturn(rc, rc);
2157 }
2158 GMMR3FreePagesCleanup(pReq);
2159 return VINF_SUCCESS;
2160}
2161
2162
2163/**
2164 * Frees all RAM during VM termination
2165 *
2166 * ASSUMES that the caller owns the PGM lock.
2167 *
2168 * @returns VBox status code.
2169 * @param pVM The cross context VM structure.
2170 */
2171int pgmR3PhysRamTerm(PVM pVM)
2172{
2173 PGM_LOCK_ASSERT_OWNER(pVM);
2174
2175 /* Reset the memory balloon. */
2176 int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2177 AssertRC(rc);
2178
2179#ifdef VBOX_WITH_PAGE_SHARING
2180 /*
2181 * Clear all registered shared modules.
2182 */
2183 pgmR3PhysAssertSharedPageChecksums(pVM);
2184 rc = GMMR3ResetSharedModules(pVM);
2185 AssertRC(rc);
2186
2187 /*
2188 * Flush the handy pages updates to make sure no shared pages are hiding
2189 * in there. (Not unlikely if the VM shuts down, apparently.)
2190 */
2191# ifdef VBOX_WITH_PGM_NEM_MODE
2192 if (!pVM->pgm.s.fNemMode)
2193# endif
2194 rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_FLUSH_HANDY_PAGES, 0, NULL);
2195#endif
2196
2197 /*
2198 * We batch up pages that should be freed instead of calling GMM for
2199 * each and every one of them.
2200 */
2201 uint32_t cPendingPages = 0;
2202 PGMMFREEPAGESREQ pReq;
2203 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2204 AssertLogRelRCReturn(rc, rc);
2205
2206 /*
2207 * Walk the ram ranges.
2208 */
2209 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2210 {
2211 uint32_t iPage = pRam->cb >> GUEST_PAGE_SHIFT;
2212 AssertMsg(((RTGCPHYS)iPage << GUEST_PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << GUEST_PAGE_SHIFT, pRam->cb));
2213
2214 while (iPage-- > 0)
2215 {
2216 PPGMPAGE pPage = &pRam->aPages[iPage];
2217 switch (PGM_PAGE_GET_TYPE(pPage))
2218 {
2219 case PGMPAGETYPE_RAM:
2220 /* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2221 /** @todo change this to explicitly free private pages here. */
2222 if (PGM_PAGE_IS_SHARED(pPage))
2223 {
2224 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage,
2225 pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), PGMPAGETYPE_RAM);
2226 AssertLogRelRCReturn(rc, rc);
2227 }
2228 break;
2229
2230 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2231 case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
2232 case PGMPAGETYPE_MMIO2:
2233 case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2234 case PGMPAGETYPE_ROM:
2235 case PGMPAGETYPE_MMIO:
2236 break;
2237 default:
2238 AssertFailed();
2239 }
2240 } /* for each page */
2241 }
2242
2243 /*
2244 * Finish off any pages pending freeing.
2245 */
2246 if (cPendingPages)
2247 {
2248 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2249 AssertLogRelRCReturn(rc, rc);
2250 }
2251 GMMR3FreePagesCleanup(pReq);
2252 return VINF_SUCCESS;
2253}
2254
2255
2256
2257/*********************************************************************************************************************************
2258* MMIO *
2259*********************************************************************************************************************************/
2260
2261/**
2262 * This is the interface IOM is using to register an MMIO region.
2263 *
2264 * It will check for conflicts and ensure that a RAM range structure
2265 * is present before calling the PGMR3HandlerPhysicalRegister API to
2266 * register the callbacks.
2267 *
2268 * @returns VBox status code.
2269 *
2270 * @param pVM The cross context VM structure.
2271 * @param GCPhys The start of the MMIO region.
2272 * @param cb The size of the MMIO region.
2273 * @param hType The physical access handler type registration.
2274 * @param uUser The user argument.
2275 * @param pszDesc The description of the MMIO region.
2276 */
2277VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMPHYSHANDLERTYPE hType,
2278 uint64_t uUser, const char *pszDesc)
2279{
2280 /*
2281 * Assert on some assumption.
2282 */
2283 VM_ASSERT_EMT(pVM);
2284 AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2285 AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2286 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2287 AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2288#ifdef VBOX_STRICT
2289 PCPGMPHYSHANDLERTYPEINT pType = pgmHandlerPhysicalTypeHandleToPtr(pVM, hType);
2290 Assert(pType);
2291 Assert(pType->enmKind == PGMPHYSHANDLERKIND_MMIO);
2292#endif
2293
2294 int rc = PGM_LOCK(pVM);
2295 AssertRCReturn(rc, rc);
2296
2297 /*
2298 * Make sure there's a RAM range structure for the region.
2299 */
2300 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2301 bool fRamExists = false;
2302 PPGMRAMRANGE pRamPrev = NULL;
2303 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2304 while (pRam && GCPhysLast >= pRam->GCPhys)
2305 {
2306 if ( GCPhysLast >= pRam->GCPhys
2307 && GCPhys <= pRam->GCPhysLast)
2308 {
2309 /* Simplification: all within the same range. */
2310 AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2311 && GCPhysLast <= pRam->GCPhysLast,
2312 ("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2313 GCPhys, GCPhysLast, pszDesc,
2314 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2315 PGM_UNLOCK(pVM),
2316 VERR_PGM_RAM_CONFLICT);
2317
2318 /* Check that it's all RAM or MMIO pages. */
2319 PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
2320 uint32_t cLeft = cb >> GUEST_PAGE_SHIFT;
2321 while (cLeft-- > 0)
2322 {
2323 AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2324 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2325 ("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2326 GCPhys, GCPhysLast, pszDesc, pRam->GCPhys, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2327 PGM_UNLOCK(pVM),
2328 VERR_PGM_RAM_CONFLICT);
2329 pPage++;
2330 }
2331
2332 /* Looks good. */
2333 fRamExists = true;
2334 break;
2335 }
2336
2337 /* next */
2338 pRamPrev = pRam;
2339 pRam = pRam->pNextR3;
2340 }
2341 PPGMRAMRANGE pNew;
2342 if (fRamExists)
2343 {
2344 pNew = NULL;
2345
2346 /*
2347 * Make all the pages in the range MMIO/ZERO pages, freeing any
2348 * RAM pages currently mapped here. This might not be 100% correct
2349 * for PCI memory, but we're doing the same thing for MMIO2 pages.
2350 */
2351 rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, NULL);
2352 AssertRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
2353
2354 /* Force a PGM pool flush as guest ram references have been changed. */
2355 /** @todo not entirely SMP safe; assuming for now the guest takes
2356 * care of this internally (not touch mapped mmio while changing the
2357 * mapping). */
2358 PVMCPU pVCpu = VMMGetCpu(pVM);
2359 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2360 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2361 }
2362 else
2363 {
2364 /*
2365 * No RAM range, insert an ad hoc one.
2366 *
2367 * Note that we don't have to tell REM about this range because
2368 * PGMHandlerPhysicalRegisterEx will do that for us.
2369 */
2370 Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2371
2372 /* Alloc. */
2373 const uint32_t cPages = cb >> GUEST_PAGE_SHIFT;
2374 const size_t cbRamRange = RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]);
2375 const size_t cRangePages = RT_ALIGN_Z(cbRamRange, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT;
2376 RTR0PTR pNewR0 = NIL_RTR0PTR;
2377 rc = SUPR3PageAllocEx(cRangePages, 0 /*fFlags*/, (void **)&pNew, &pNewR0, NULL /*paPages*/);
2378 AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), PGM_UNLOCK(pVM), rc);
2379
2380#ifdef VBOX_WITH_NATIVE_NEM
2381 /* Notify NEM. */
2382 uint8_t u2State = 0; /* (must have valid state as there can't be anything to preserve) */
2383 if (VM_IS_NEM_ENABLED(pVM))
2384 {
2385 rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, cPages << GUEST_PAGE_SHIFT, 0 /*fFlags*/, NULL, NULL,
2386 &u2State, &pNew->uNemRange);
2387 AssertLogRelRCReturnStmt(rc, SUPR3PageFreeEx(pNew, cRangePages), rc);
2388 }
2389#endif
2390
2391 /* Initialize the range. */
2392 pNew->pSelfR0 = pNewR0;
2393 pNew->GCPhys = GCPhys;
2394 pNew->GCPhysLast = GCPhysLast;
2395 pNew->cb = cb;
2396 pNew->pszDesc = pszDesc;
2397 pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2398 pNew->pvR3 = NULL;
2399 pNew->paLSPages = NULL;
2400
2401 uint32_t iPage = cPages;
2402 while (iPage-- > 0)
2403 {
2404 PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2405#ifdef VBOX_WITH_NATIVE_NEM
2406 PGM_PAGE_SET_NEM_STATE(&pNew->aPages[iPage], u2State);
2407#endif
2408 }
2409 Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2410
2411 /* update the page count stats. */
2412 pVM->pgm.s.cPureMmioPages += cPages;
2413 pVM->pgm.s.cAllPages += cPages;
2414
2415 /* link it */
2416 pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2417 }
2418
2419 /*
2420 * Register the access handler.
2421 */
2422 rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, hType, uUser, pszDesc);
2423 if (RT_SUCCESS(rc))
2424 {
2425#ifdef VBOX_WITH_NATIVE_NEM
2426 /* Late NEM notification. */
2427 if (VM_IS_NEM_ENABLED(pVM))
2428 {
2429 uint32_t const fNemNotify = (fRamExists ? NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE : 0);
2430 rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify,
2431 fRamExists ? (uint8_t *)pRam->pvR3 + (uintptr_t)(GCPhys - pRam->GCPhys) : NULL,
2432 NULL, !fRamExists ? &pRam->uNemRange : NULL);
2433 AssertLogRelRCReturn(rc, rc);
2434 }
2435#endif
2436 }
2437 /** @todo the phys handler failure handling isn't complete, esp. wrt NEM. */
2438 else if (!fRamExists)
2439 {
2440 pVM->pgm.s.cPureMmioPages -= cb >> GUEST_PAGE_SHIFT;
2441 pVM->pgm.s.cAllPages -= cb >> GUEST_PAGE_SHIFT;
2442
2443 /* remove the ad hoc range. */
2444 pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2445 pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2446 SUPR3PageFreeEx(pRam, RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cb >> GUEST_PAGE_SHIFT]),
2447 HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT);
2448 }
2449 pgmPhysInvalidatePageMapTLB(pVM);
2450
2451 PGM_UNLOCK(pVM);
2452 return rc;
2453}
2454
2455
2456/**
2457 * This is the interface IOM is using to register an MMIO region.
2458 *
2459 * It will take care of calling PGMHandlerPhysicalDeregister and clean up
2460 * any ad hoc PGMRAMRANGE left behind.
2461 *
2462 * @returns VBox status code.
2463 * @param pVM The cross context VM structure.
2464 * @param GCPhys The start of the MMIO region.
2465 * @param cb The size of the MMIO region.
2466 */
2467VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2468{
2469 VM_ASSERT_EMT(pVM);
2470
2471 int rc = PGM_LOCK(pVM);
2472 AssertRCReturn(rc, rc);
2473
2474 /*
2475 * First deregister the handler, then check if we should remove the ram range.
2476 */
2477 rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2478 if (RT_SUCCESS(rc))
2479 {
2480 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2481 PPGMRAMRANGE pRamPrev = NULL;
2482 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2483 while (pRam && GCPhysLast >= pRam->GCPhys)
2484 {
2485 /** @todo We're being a bit too careful here. rewrite. */
2486 if ( GCPhysLast == pRam->GCPhysLast
2487 && GCPhys == pRam->GCPhys)
2488 {
2489 Assert(pRam->cb == cb);
2490
2491 /*
2492 * See if all the pages are dead MMIO pages.
2493 */
2494 uint32_t const cGuestPages = cb >> GUEST_PAGE_SHIFT;
2495 bool fAllMMIO = true;
2496 uint32_t iPage = 0;
2497 uint32_t cLeft = cGuestPages;
2498 while (cLeft-- > 0)
2499 {
2500 PPGMPAGE pPage = &pRam->aPages[iPage];
2501 if ( !PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)
2502 /*|| not-out-of-action later */)
2503 {
2504 fAllMMIO = false;
2505 AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), pPage));
2506 break;
2507 }
2508 Assert( PGM_PAGE_IS_ZERO(pPage)
2509 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2510 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO);
2511 pPage++;
2512 }
2513 if (fAllMMIO)
2514 {
2515 /*
2516 * Ad-hoc range, unlink and free it.
2517 */
2518 Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2519 GCPhys, GCPhysLast, pRam->pszDesc));
2520 /** @todo check the ad-hoc flags? */
2521
2522#ifdef VBOX_WITH_NATIVE_NEM
2523 if (VM_IS_NEM_ENABLED(pVM)) /* Notify REM before we unlink the range. */
2524 {
2525 rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, GCPhysLast - GCPhys + 1, 0 /*fFlags*/,
2526 NULL, NULL, NULL, &pRam->uNemRange);
2527 AssertLogRelRCReturn(rc, rc);
2528 }
2529#endif
2530
2531 pVM->pgm.s.cAllPages -= cGuestPages;
2532 pVM->pgm.s.cPureMmioPages -= cGuestPages;
2533
2534 pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2535 const uint32_t cPages = pRam->cb >> GUEST_PAGE_SHIFT;
2536 const size_t cbRamRange = RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]);
2537 pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2538 SUPR3PageFreeEx(pRam, RT_ALIGN_Z(cbRamRange, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT);
2539 break;
2540 }
2541 }
2542
2543 /*
2544 * Range match? It will all be within one range (see PGMAllHandler.cpp).
2545 */
2546 if ( GCPhysLast >= pRam->GCPhys
2547 && GCPhys <= pRam->GCPhysLast)
2548 {
2549 Assert(GCPhys >= pRam->GCPhys);
2550 Assert(GCPhysLast <= pRam->GCPhysLast);
2551
2552 /*
2553 * Turn the pages back into RAM pages.
2554 */
2555 uint32_t iPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
2556 uint32_t cLeft = cb >> GUEST_PAGE_SHIFT;
2557 while (cLeft--)
2558 {
2559 PPGMPAGE pPage = &pRam->aPages[iPage];
2560 AssertMsg( (PGM_PAGE_IS_MMIO(pPage) && PGM_PAGE_IS_ZERO(pPage))
2561 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2562 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO,
2563 ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), pPage));
2564 if (PGM_PAGE_IS_MMIO_OR_ALIAS(pPage))
2565 PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2566 iPage++;
2567 }
2568
2569#ifdef VBOX_WITH_NATIVE_NEM
2570 /* Notify REM (failure will probably leave things in a non-working state). */
2571 if (VM_IS_NEM_ENABLED(pVM))
2572 {
2573 uint8_t u2State = UINT8_MAX;
2574 rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, GCPhysLast - GCPhys + 1, NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
2575 pRam->pvR3 ? (uint8_t *)pRam->pvR3 + GCPhys - pRam->GCPhys : NULL,
2576 NULL, &u2State, &pRam->uNemRange);
2577 AssertLogRelRCReturn(rc, rc);
2578 if (u2State != UINT8_MAX)
2579 pgmPhysSetNemStateForPages(&pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT],
2580 cb >> GUEST_PAGE_SHIFT, u2State);
2581 }
2582#endif
2583 break;
2584 }
2585
2586 /* next */
2587 pRamPrev = pRam;
2588 pRam = pRam->pNextR3;
2589 }
2590 }
2591
2592 /* Force a PGM pool flush as guest ram references have been changed. */
2593 /** @todo Not entirely SMP safe; assuming for now the guest takes care of
2594 * this internally (not touch mapped mmio while changing the mapping). */
2595 PVMCPU pVCpu = VMMGetCpu(pVM);
2596 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2597 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2598
2599 pgmPhysInvalidatePageMapTLB(pVM);
2600 pgmPhysInvalidRamRangeTlbs(pVM);
2601 PGM_UNLOCK(pVM);
2602 return rc;
2603}
2604
2605
2606
2607/*********************************************************************************************************************************
2608* MMIO2 *
2609*********************************************************************************************************************************/
2610
2611/**
2612 * Locate a MMIO2 range.
2613 *
2614 * @returns Pointer to the MMIO2 range.
2615 * @param pVM The cross context VM structure.
2616 * @param pDevIns The device instance owning the region.
2617 * @param iSubDev The sub-device number.
2618 * @param iRegion The region.
2619 * @param hMmio2 Handle to look up. If NIL, use the @a iSubDev and
2620 * @a iRegion.
2621 */
2622DECLINLINE(PPGMREGMMIO2RANGE) pgmR3PhysMmio2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev,
2623 uint32_t iRegion, PGMMMIO2HANDLE hMmio2)
2624{
2625 if (hMmio2 != NIL_PGMMMIO2HANDLE)
2626 {
2627 if (hMmio2 <= RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3) && hMmio2 != 0)
2628 {
2629 PPGMREGMMIO2RANGE pCur = pVM->pgm.s.apMmio2RangesR3[hMmio2 - 1];
2630 if (pCur && pCur->pDevInsR3 == pDevIns)
2631 {
2632 Assert(pCur->idMmio2 == hMmio2);
2633 AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, NULL);
2634 return pCur;
2635 }
2636 Assert(!pCur);
2637 }
2638 for (PPGMREGMMIO2RANGE pCur = pVM->pgm.s.pRegMmioRangesR3; pCur; pCur = pCur->pNextR3)
2639 if (pCur->idMmio2 == hMmio2)
2640 {
2641 AssertBreak(pCur->pDevInsR3 == pDevIns);
2642 AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, NULL);
2643 return pCur;
2644 }
2645 }
2646 else
2647 {
2648 /*
2649 * Search the list. There shouldn't be many entries.
2650 */
2651 /** @todo Optimize this lookup! There may now be many entries and it'll
2652 * become really slow when doing MMR3HyperMapMMIO2 and similar. */
2653 for (PPGMREGMMIO2RANGE pCur = pVM->pgm.s.pRegMmioRangesR3; pCur; pCur = pCur->pNextR3)
2654 if ( pCur->pDevInsR3 == pDevIns
2655 && pCur->iRegion == iRegion
2656 && pCur->iSubDev == iSubDev)
2657 return pCur;
2658 }
2659 return NULL;
2660}
2661
2662
2663/**
2664 * Worker for PGMR3PhysMmio2ControlDirtyPageTracking and PGMR3PhysMmio2Map.
2665 */
2666static int pgmR3PhysMmio2EnableDirtyPageTracing(PVM pVM, PPGMREGMMIO2RANGE pFirstMmio2)
2667{
2668 int rc = VINF_SUCCESS;
2669 for (PPGMREGMMIO2RANGE pCurMmio2 = pFirstMmio2; pCurMmio2; pCurMmio2 = pCurMmio2->pNextR3)
2670 {
2671 Assert(!(pCurMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_TRACKING));
2672 int rc2 = pgmHandlerPhysicalExRegister(pVM, pCurMmio2->pPhysHandlerR3, pCurMmio2->RamRange.GCPhys,
2673 pCurMmio2->RamRange.GCPhysLast);
2674 AssertLogRelMsgRC(rc2, ("%#RGp-%#RGp %s failed -> %Rrc\n", pCurMmio2->RamRange.GCPhys, pCurMmio2->RamRange.GCPhysLast,
2675 pCurMmio2->RamRange.pszDesc, rc2));
2676 if (RT_SUCCESS(rc2))
2677 pCurMmio2->fFlags |= PGMREGMMIO2RANGE_F_IS_TRACKING;
2678 else if (RT_SUCCESS(rc))
2679 rc = rc2;
2680 if (pCurMmio2->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
2681 return rc;
2682 }
2683 AssertFailed();
2684 return rc;
2685}
2686
2687
2688/**
2689 * Worker for PGMR3PhysMmio2ControlDirtyPageTracking and PGMR3PhysMmio2Unmap.
2690 */
2691static int pgmR3PhysMmio2DisableDirtyPageTracing(PVM pVM, PPGMREGMMIO2RANGE pFirstMmio2)
2692{
2693 for (PPGMREGMMIO2RANGE pCurMmio2 = pFirstMmio2; pCurMmio2; pCurMmio2 = pCurMmio2->pNextR3)
2694 {
2695 if (pCurMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_TRACKING)
2696 {
2697 int rc2 = pgmHandlerPhysicalExDeregister(pVM, pCurMmio2->pPhysHandlerR3);
2698 AssertLogRelMsgRC(rc2, ("%#RGp-%#RGp %s failed -> %Rrc\n", pCurMmio2->RamRange.GCPhys, pCurMmio2->RamRange.GCPhysLast,
2699 pCurMmio2->RamRange.pszDesc, rc2));
2700 pCurMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_IS_TRACKING;
2701 }
2702 if (pCurMmio2->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
2703 return VINF_SUCCESS;
2704 }
2705 AssertFailed();
2706 return VINF_SUCCESS;
2707
2708}
2709
2710
2711/**
2712 * Calculates the number of chunks
2713 *
2714 * @returns Number of registration chunk needed.
2715 * @param pVM The cross context VM structure.
2716 * @param cb The size of the MMIO/MMIO2 range.
2717 * @param pcPagesPerChunk Where to return the number of pages tracked by each
2718 * chunk. Optional.
2719 * @param pcbChunk Where to return the guest mapping size for a chunk.
2720 */
2721static uint16_t pgmR3PhysMmio2CalcChunkCount(PVM pVM, RTGCPHYS cb, uint32_t *pcPagesPerChunk, uint32_t *pcbChunk)
2722{
2723 RT_NOREF_PV(pVM); /* without raw mode */
2724
2725 /*
2726 * This is the same calculation as PGMR3PhysRegisterRam does, except we'll be
2727 * needing a few bytes extra the PGMREGMMIO2RANGE structure.
2728 *
2729 * Note! In additions, we've got a 24 bit sub-page range for MMIO2 ranges, leaving
2730 * us with an absolute maximum of 16777215 pages per chunk (close to 64 GB).
2731 */
2732 uint32_t const cPagesPerChunk = _4M;
2733 Assert(RT_ALIGN_32(cPagesPerChunk, X86_PD_PAE_SHIFT - X86_PAGE_SHIFT)); /* NEM large page requirement: 1GB pages. */
2734 uint32_t const cbChunk = RT_UOFFSETOF_DYN(PGMREGMMIO2RANGE, RamRange.aPages[cPagesPerChunk]);
2735 AssertRelease(cPagesPerChunk < _16M);
2736
2737 if (pcbChunk)
2738 *pcbChunk = cbChunk;
2739 if (pcPagesPerChunk)
2740 *pcPagesPerChunk = cPagesPerChunk;
2741
2742 /* Calc the number of chunks we need. */
2743 RTGCPHYS const cGuestPages = cb >> GUEST_PAGE_SHIFT;
2744 uint16_t cChunks = (uint16_t)((cGuestPages + cPagesPerChunk - 1) / cPagesPerChunk);
2745 AssertRelease((RTGCPHYS)cChunks * cPagesPerChunk >= cGuestPages);
2746 return cChunks;
2747}
2748
2749
2750/**
2751 * Worker for PGMR3PhysMMIO2Register that allocates and the PGMREGMMIO2RANGE
2752 * structures and does basic initialization.
2753 *
2754 * Caller must set type specfic members and initialize the PGMPAGE structures.
2755 *
2756 * This was previously also used by PGMR3PhysMmio2PreRegister, a function for
2757 * pre-registering MMIO that was later (6.1) replaced by a new handle based IOM
2758 * interface. The reference to caller and type above is purely historical.
2759 *
2760 * @returns VBox status code.
2761 * @param pVM The cross context VM structure.
2762 * @param pDevIns The device instance owning the region.
2763 * @param iSubDev The sub-device number (internal PCI config number).
2764 * @param iRegion The region number. If the MMIO2 memory is a PCI
2765 * I/O region this number has to be the number of that
2766 * region. Otherwise it can be any number safe
2767 * UINT8_MAX.
2768 * @param cb The size of the region. Must be page aligned.
2769 * @param fFlags PGMPHYS_MMIO2_FLAGS_XXX.
2770 * @param idMmio2 The MMIO2 ID for the first chunk.
2771 * @param pszDesc The description.
2772 * @param ppHeadRet Where to return the pointer to the first
2773 * registration chunk.
2774 *
2775 * @thread EMT
2776 */
2777static int pgmR3PhysMmio2Create(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags,
2778 uint8_t idMmio2, const char *pszDesc, PPGMREGMMIO2RANGE *ppHeadRet)
2779{
2780 /*
2781 * Figure out how many chunks we need and of which size.
2782 */
2783 uint32_t cPagesPerChunk;
2784 uint16_t cChunks = pgmR3PhysMmio2CalcChunkCount(pVM, cb, &cPagesPerChunk, NULL);
2785 AssertReturn(cChunks, VERR_PGM_PHYS_MMIO_EX_IPE);
2786
2787 /*
2788 * Allocate the chunks.
2789 */
2790 PPGMREGMMIO2RANGE *ppNext = ppHeadRet;
2791 *ppNext = NULL;
2792
2793 int rc = VINF_SUCCESS;
2794 uint32_t cPagesLeft = cb >> GUEST_PAGE_SHIFT;
2795 for (uint16_t iChunk = 0; iChunk < cChunks && RT_SUCCESS(rc); iChunk++, idMmio2++)
2796 {
2797 /*
2798 * We currently do a single RAM range for the whole thing. This will
2799 * probably have to change once someone needs really large MMIO regions,
2800 * as we will be running into SUPR3PageAllocEx limitations and such.
2801 */
2802 const uint32_t cPagesTrackedByChunk = RT_MIN(cPagesLeft, cPagesPerChunk);
2803 const size_t cbRange = RT_UOFFSETOF_DYN(PGMREGMMIO2RANGE, RamRange.aPages[cPagesTrackedByChunk]);
2804 PPGMREGMMIO2RANGE pNew = NULL;
2805
2806 /*
2807 * Allocate memory for the registration structure.
2808 */
2809 size_t const cChunkPages = RT_ALIGN_Z(cbRange, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT;
2810 size_t const cbChunk = (1 + cChunkPages + 1) << HOST_PAGE_SHIFT;
2811 AssertLogRelBreakStmt(cbChunk == (uint32_t)cbChunk, rc = VERR_OUT_OF_RANGE);
2812 RTR0PTR R0PtrChunk = NIL_RTR0PTR;
2813 void *pvChunk = NULL;
2814 rc = SUPR3PageAllocEx(cChunkPages, 0 /*fFlags*/, &pvChunk, &R0PtrChunk, NULL /*paPages*/);
2815 AssertLogRelMsgRCBreak(rc, ("rc=%Rrc, cChunkPages=%#zx\n", rc, cChunkPages));
2816
2817 Assert(R0PtrChunk != NIL_RTR0PTR || PGM_IS_IN_NEM_MODE(pVM));
2818 RT_BZERO(pvChunk, cChunkPages << HOST_PAGE_SHIFT);
2819
2820 pNew = (PPGMREGMMIO2RANGE)pvChunk;
2821 pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_FLOATING;
2822 pNew->RamRange.pSelfR0 = R0PtrChunk + RT_UOFFSETOF(PGMREGMMIO2RANGE, RamRange);
2823
2824 /*
2825 * Initialize the registration structure (caller does specific bits).
2826 */
2827 pNew->pDevInsR3 = pDevIns;
2828 //pNew->pvR3 = NULL;
2829 //pNew->pNext = NULL;
2830 if (iChunk == 0)
2831 pNew->fFlags |= PGMREGMMIO2RANGE_F_FIRST_CHUNK;
2832 if (iChunk + 1 == cChunks)
2833 pNew->fFlags |= PGMREGMMIO2RANGE_F_LAST_CHUNK;
2834 if (fFlags & PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES)
2835 pNew->fFlags |= PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES;
2836 pNew->iSubDev = iSubDev;
2837 pNew->iRegion = iRegion;
2838 pNew->idSavedState = UINT8_MAX;
2839 pNew->idMmio2 = idMmio2;
2840 //pNew->pPhysHandlerR3 = NULL;
2841 //pNew->paLSPages = NULL;
2842 pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2843 pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2844 pNew->RamRange.pszDesc = pszDesc;
2845 pNew->RamRange.cb = pNew->cbReal = (RTGCPHYS)cPagesTrackedByChunk << X86_PAGE_SHIFT;
2846 pNew->RamRange.fFlags |= PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX;
2847 pNew->RamRange.uNemRange = UINT32_MAX;
2848 //pNew->RamRange.pvR3 = NULL;
2849 //pNew->RamRange.paLSPages = NULL;
2850
2851 *ppNext = pNew;
2852 ASMCompilerBarrier();
2853 cPagesLeft -= cPagesTrackedByChunk;
2854 ppNext = &pNew->pNextR3;
2855
2856 /*
2857 * Pre-allocate a handler if we're tracking dirty pages, unless NEM takes care of this.
2858 */
2859 if ( (fFlags & PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES)
2860#ifdef VBOX_WITH_PGM_NEM_MODE
2861 && (!VM_IS_NEM_ENABLED(pVM) || !NEMR3IsMmio2DirtyPageTrackingSupported(pVM))
2862#endif
2863 )
2864
2865 {
2866 rc = pgmHandlerPhysicalExCreate(pVM, pVM->pgm.s.hMmio2DirtyPhysHandlerType, idMmio2, pszDesc, &pNew->pPhysHandlerR3);
2867 AssertLogRelMsgRCBreak(rc, ("idMmio2=%zu\n", idMmio2));
2868 }
2869 }
2870 Assert(cPagesLeft == 0);
2871
2872 if (RT_SUCCESS(rc))
2873 {
2874 Assert((*ppHeadRet)->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK);
2875 return VINF_SUCCESS;
2876 }
2877
2878 /*
2879 * Free floating ranges.
2880 */
2881 while (*ppHeadRet)
2882 {
2883 PPGMREGMMIO2RANGE pFree = *ppHeadRet;
2884 *ppHeadRet = pFree->pNextR3;
2885
2886 if (pFree->pPhysHandlerR3)
2887 {
2888 pgmHandlerPhysicalExDestroy(pVM, pFree->pPhysHandlerR3);
2889 pFree->pPhysHandlerR3 = NULL;
2890 }
2891
2892 if (pFree->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING)
2893 {
2894 const size_t cbRange = RT_UOFFSETOF_DYN(PGMREGMMIO2RANGE,
2895 RamRange.aPages[pFree->RamRange.cb >> X86_PAGE_SHIFT]);
2896 size_t const cChunkPages = RT_ALIGN_Z(cbRange, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT;
2897 SUPR3PageFreeEx(pFree, cChunkPages);
2898 }
2899 }
2900
2901 return rc;
2902}
2903
2904
2905/**
2906 * Common worker PGMR3PhysMmio2PreRegister & PGMR3PhysMMIO2Register that links a
2907 * complete registration entry into the lists and lookup tables.
2908 *
2909 * @param pVM The cross context VM structure.
2910 * @param pNew The new MMIO / MMIO2 registration to link.
2911 */
2912static void pgmR3PhysMmio2Link(PVM pVM, PPGMREGMMIO2RANGE pNew)
2913{
2914 Assert(pNew->idMmio2 != UINT8_MAX);
2915
2916 /*
2917 * Link it into the list (order doesn't matter, so insert it at the head).
2918 *
2919 * Note! The range we're linking may consist of multiple chunks, so we
2920 * have to find the last one.
2921 */
2922 PPGMREGMMIO2RANGE pLast = pNew;
2923 for (pLast = pNew; ; pLast = pLast->pNextR3)
2924 {
2925 if (pLast->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
2926 break;
2927 Assert(pLast->pNextR3);
2928 Assert(pLast->pNextR3->pDevInsR3 == pNew->pDevInsR3);
2929 Assert(pLast->pNextR3->iSubDev == pNew->iSubDev);
2930 Assert(pLast->pNextR3->iRegion == pNew->iRegion);
2931 Assert(pLast->pNextR3->idMmio2 == pLast->idMmio2 + 1);
2932 }
2933
2934 PGM_LOCK_VOID(pVM);
2935
2936 /* Link in the chain of ranges at the head of the list. */
2937 pLast->pNextR3 = pVM->pgm.s.pRegMmioRangesR3;
2938 pVM->pgm.s.pRegMmioRangesR3 = pNew;
2939
2940 /* Insert the MMIO2 range/page IDs. */
2941 uint8_t idMmio2 = pNew->idMmio2;
2942 for (;;)
2943 {
2944 Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == NULL);
2945 Assert(pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] == NIL_RTR0PTR);
2946 pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = pNew;
2947 pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = pNew->RamRange.pSelfR0 - RT_UOFFSETOF(PGMREGMMIO2RANGE, RamRange);
2948 if (pNew->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
2949 break;
2950 pNew = pNew->pNextR3;
2951 idMmio2++;
2952 }
2953
2954 pgmPhysInvalidatePageMapTLB(pVM);
2955 PGM_UNLOCK(pVM);
2956}
2957
2958
2959/**
2960 * Allocate and register an MMIO2 region.
2961 *
2962 * As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2963 * associated with a device. It is also non-shared memory with a permanent
2964 * ring-3 mapping and page backing (presently).
2965 *
2966 * A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2967 * the VM, in which case we'll drop the base memory pages. Presently we will
2968 * make no attempt to preserve anything that happens to be present in the base
2969 * memory that is replaced, this is of course incorrect but it's too much
2970 * effort.
2971 *
2972 * @returns VBox status code.
2973 * @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2974 * memory.
2975 * @retval VERR_ALREADY_EXISTS if the region already exists.
2976 *
2977 * @param pVM The cross context VM structure.
2978 * @param pDevIns The device instance owning the region.
2979 * @param iSubDev The sub-device number.
2980 * @param iRegion The region number. If the MMIO2 memory is a PCI
2981 * I/O region this number has to be the number of that
2982 * region. Otherwise it can be any number save
2983 * UINT8_MAX.
2984 * @param cb The size of the region. Must be page aligned.
2985 * @param fFlags Reserved for future use, must be zero.
2986 * @param pszDesc The description.
2987 * @param ppv Where to store the pointer to the ring-3 mapping of
2988 * the memory.
2989 * @param phRegion Where to return the MMIO2 region handle. Optional.
2990 * @thread EMT
2991 */
2992VMMR3_INT_DECL(int) PGMR3PhysMmio2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cb,
2993 uint32_t fFlags, const char *pszDesc, void **ppv, PGMMMIO2HANDLE *phRegion)
2994{
2995 /*
2996 * Validate input.
2997 */
2998 AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2999 *ppv = NULL;
3000 if (phRegion)
3001 {
3002 AssertPtrReturn(phRegion, VERR_INVALID_POINTER);
3003 *phRegion = NIL_PGMMMIO2HANDLE;
3004 }
3005 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3006 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3007 AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
3008 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3009 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3010 AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
3011 AssertReturn(pgmR3PhysMmio2Find(pVM, pDevIns, iSubDev, iRegion, NIL_PGMMMIO2HANDLE) == NULL, VERR_ALREADY_EXISTS);
3012 AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3013 AssertReturn(cb, VERR_INVALID_PARAMETER);
3014 AssertReturn(!(fFlags & ~PGMPHYS_MMIO2_FLAGS_VALID_MASK), VERR_INVALID_FLAGS);
3015
3016 const uint32_t cGuestPages = cb >> GUEST_PAGE_SHIFT;
3017 AssertLogRelReturn(((RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
3018 AssertLogRelReturn(cGuestPages <= (MM_MMIO_64_MAX >> X86_PAGE_SHIFT), VERR_OUT_OF_RANGE);
3019 AssertLogRelReturn(cGuestPages <= PGM_MMIO2_MAX_PAGE_COUNT, VERR_OUT_OF_RANGE);
3020
3021 /*
3022 * For the 2nd+ instance, mangle the description string so it's unique.
3023 */
3024 if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
3025 {
3026 pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
3027 if (!pszDesc)
3028 return VERR_NO_MEMORY;
3029 }
3030
3031 /*
3032 * Allocate an MMIO2 range ID (not freed on failure).
3033 *
3034 * The zero ID is not used as it could be confused with NIL_GMM_PAGEID, so
3035 * the IDs goes from 1 thru PGM_MMIO2_MAX_RANGES.
3036 */
3037 unsigned cChunks = pgmR3PhysMmio2CalcChunkCount(pVM, cb, NULL, NULL);
3038
3039 PGM_LOCK_VOID(pVM);
3040 AssertCompile(PGM_MMIO2_MAX_RANGES < 255);
3041 uint8_t const idMmio2 = pVM->pgm.s.cMmio2Regions + 1;
3042 unsigned const cNewMmio2Regions = pVM->pgm.s.cMmio2Regions + cChunks;
3043 if (cNewMmio2Regions > PGM_MMIO2_MAX_RANGES)
3044 {
3045 PGM_UNLOCK(pVM);
3046 AssertLogRelFailedReturn(VERR_PGM_TOO_MANY_MMIO2_RANGES);
3047 }
3048 pVM->pgm.s.cMmio2Regions = cNewMmio2Regions;
3049 PGM_UNLOCK(pVM);
3050
3051 /*
3052 * Try reserve and allocate the backing memory first as this is what is
3053 * most likely to fail.
3054 */
3055 int rc = MMR3AdjustFixedReservation(pVM, cGuestPages, pszDesc);
3056 if (RT_SUCCESS(rc))
3057 {
3058 const uint32_t cHostPages = RT_ALIGN_T(cb, HOST_PAGE_SIZE, RTGCPHYS) >> HOST_PAGE_SHIFT;
3059 PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cHostPages * sizeof(SUPPAGE));
3060 if (RT_SUCCESS(rc))
3061 {
3062 void *pvPages = NULL;
3063#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
3064 RTR0PTR pvPagesR0 = NIL_RTR0PTR;
3065#endif
3066#ifdef VBOX_WITH_PGM_NEM_MODE
3067 if (PGM_IS_IN_NEM_MODE(pVM))
3068 rc = SUPR3PageAlloc(cHostPages, pVM->pgm.s.fUseLargePages ? SUP_PAGE_ALLOC_F_LARGE_PAGES : 0, &pvPages);
3069 else
3070#endif
3071 {
3072#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
3073 rc = SUPR3PageAllocEx(cHostPages, 0 /*fFlags*/, &pvPages, &pvPagesR0, paPages);
3074#else
3075 rc = SUPR3PageAllocEx(cHostPages, 0 /*fFlags*/, &pvPages, NULL /*pR0Ptr*/, paPages);
3076#endif
3077 }
3078 if (RT_SUCCESS(rc))
3079 {
3080 memset(pvPages, 0, cGuestPages * GUEST_PAGE_SIZE);
3081
3082 /*
3083 * Create the registered MMIO range record for it.
3084 */
3085 PPGMREGMMIO2RANGE pNew;
3086 rc = pgmR3PhysMmio2Create(pVM, pDevIns, iSubDev, iRegion, cb, fFlags, idMmio2, pszDesc, &pNew);
3087 if (RT_SUCCESS(rc))
3088 {
3089 if (phRegion)
3090 *phRegion = idMmio2; /* The ID of the first chunk. */
3091
3092 uint32_t iSrcPage = 0;
3093 uint8_t *pbCurPages = (uint8_t *)pvPages;
3094 for (PPGMREGMMIO2RANGE pCur = pNew; pCur; pCur = pCur->pNextR3)
3095 {
3096 pCur->pvR3 = pbCurPages;
3097#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
3098 pCur->pvR0 = pvPagesR0 + (iSrcPage << GUEST_PAGE_SHIFT);
3099#endif
3100 pCur->RamRange.pvR3 = pbCurPages;
3101
3102 uint32_t iDstPage = pCur->RamRange.cb >> GUEST_PAGE_SHIFT;
3103#ifdef VBOX_WITH_PGM_NEM_MODE
3104 if (PGM_IS_IN_NEM_MODE(pVM))
3105 while (iDstPage-- > 0)
3106 PGM_PAGE_INIT(&pNew->RamRange.aPages[iDstPage], UINT64_C(0x0000ffffffff0000),
3107 PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
3108 PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
3109 else
3110#endif
3111 {
3112 AssertRelease(HOST_PAGE_SHIFT == GUEST_PAGE_SHIFT);
3113 while (iDstPage-- > 0)
3114 PGM_PAGE_INIT(&pNew->RamRange.aPages[iDstPage], paPages[iDstPage + iSrcPage].Phys,
3115 PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
3116 PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
3117 }
3118
3119 /* advance. */
3120 iSrcPage += pCur->RamRange.cb >> GUEST_PAGE_SHIFT;
3121 pbCurPages += pCur->RamRange.cb;
3122 }
3123
3124 RTMemTmpFree(paPages);
3125
3126 /*
3127 * Update the page count stats, link the registration and we're done.
3128 */
3129 pVM->pgm.s.cAllPages += cGuestPages;
3130 pVM->pgm.s.cPrivatePages += cGuestPages;
3131
3132 pgmR3PhysMmio2Link(pVM, pNew);
3133
3134 *ppv = pvPages;
3135 return VINF_SUCCESS;
3136 }
3137
3138 SUPR3PageFreeEx(pvPages, cHostPages);
3139 }
3140 }
3141 RTMemTmpFree(paPages);
3142 MMR3AdjustFixedReservation(pVM, -(int32_t)cGuestPages, pszDesc);
3143 }
3144 if (pDevIns->iInstance > 0)
3145 MMR3HeapFree((void *)pszDesc);
3146 return rc;
3147}
3148
3149
3150/**
3151 * Deregisters and frees an MMIO2 region.
3152 *
3153 * Any physical access handlers registered for the region must be deregistered
3154 * before calling this function.
3155 *
3156 * @returns VBox status code.
3157 * @param pVM The cross context VM structure.
3158 * @param pDevIns The device instance owning the region.
3159 * @param hMmio2 The MMIO2 handle to deregister, or NIL if all
3160 * regions for the given device is to be deregistered.
3161 */
3162VMMR3_INT_DECL(int) PGMR3PhysMmio2Deregister(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
3163{
3164 /*
3165 * Validate input.
3166 */
3167 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3168 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3169
3170 /*
3171 * The loop here scanning all registrations will make sure that multi-chunk ranges
3172 * get properly deregistered, though it's original purpose was the wildcard iRegion.
3173 */
3174 PGM_LOCK_VOID(pVM);
3175 int rc = VINF_SUCCESS;
3176 unsigned cFound = 0;
3177 PPGMREGMMIO2RANGE pPrev = NULL;
3178 PPGMREGMMIO2RANGE pCur = pVM->pgm.s.pRegMmioRangesR3;
3179 while (pCur)
3180 {
3181 uint32_t const fFlags = pCur->fFlags;
3182 if ( pCur->pDevInsR3 == pDevIns
3183 && ( hMmio2 == NIL_PGMMMIO2HANDLE
3184 || pCur->idMmio2 == hMmio2))
3185 {
3186 cFound++;
3187
3188 /*
3189 * Unmap it if it's mapped.
3190 */
3191 if (fFlags & PGMREGMMIO2RANGE_F_MAPPED)
3192 {
3193 int rc2 = PGMR3PhysMmio2Unmap(pVM, pCur->pDevInsR3, pCur->idMmio2, pCur->RamRange.GCPhys);
3194 AssertRC(rc2);
3195 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3196 rc = rc2;
3197 }
3198
3199 /*
3200 * Unlink it
3201 */
3202 PPGMREGMMIO2RANGE pNext = pCur->pNextR3;
3203 if (pPrev)
3204 pPrev->pNextR3 = pNext;
3205 else
3206 pVM->pgm.s.pRegMmioRangesR3 = pNext;
3207 pCur->pNextR3 = NULL;
3208
3209 uint8_t idMmio2 = pCur->idMmio2;
3210 Assert(idMmio2 <= RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3));
3211 if (idMmio2 <= RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3))
3212 {
3213 Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == pCur);
3214 pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = NULL;
3215 pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = NIL_RTR0PTR;
3216 }
3217
3218 /*
3219 * Free the memory.
3220 */
3221 uint32_t const cGuestPages = pCur->cbReal >> GUEST_PAGE_SHIFT;
3222 uint32_t const cHostPages = RT_ALIGN_T(pCur->cbReal, HOST_PAGE_SIZE, RTGCPHYS) >> HOST_PAGE_SHIFT;
3223#ifdef VBOX_WITH_PGM_NEM_MODE
3224 if (!pVM->pgm.s.fNemMode)
3225#endif
3226 {
3227 int rc2 = SUPR3PageFreeEx(pCur->pvR3, cHostPages);
3228 AssertRC(rc2);
3229 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3230 rc = rc2;
3231
3232 rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cGuestPages, pCur->RamRange.pszDesc);
3233 AssertRC(rc2);
3234 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3235 rc = rc2;
3236 }
3237#ifdef VBOX_WITH_PGM_NEM_MODE
3238 else
3239 {
3240 int rc2 = SUPR3PageFreeEx(pCur->pvR3, cHostPages);
3241 AssertRC(rc2);
3242 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3243 rc = rc2;
3244 }
3245#endif
3246
3247 if (pCur->pPhysHandlerR3)
3248 {
3249 pgmHandlerPhysicalExDestroy(pVM, pCur->pPhysHandlerR3);
3250 pCur->pPhysHandlerR3 = NULL;
3251 }
3252
3253 /* we're leaking hyper memory here if done at runtime. */
3254#ifdef VBOX_STRICT
3255 VMSTATE const enmState = VMR3GetState(pVM);
3256 AssertMsg( enmState == VMSTATE_POWERING_OFF
3257 || enmState == VMSTATE_POWERING_OFF_LS
3258 || enmState == VMSTATE_OFF
3259 || enmState == VMSTATE_OFF_LS
3260 || enmState == VMSTATE_DESTROYING
3261 || enmState == VMSTATE_TERMINATED
3262 || enmState == VMSTATE_CREATING
3263 , ("%s\n", VMR3GetStateName(enmState)));
3264#endif
3265
3266 if (pCur->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING)
3267 {
3268 const size_t cbRange = RT_UOFFSETOF_DYN(PGMREGMMIO2RANGE, RamRange.aPages[cGuestPages]);
3269 size_t const cChunkPages = RT_ALIGN_Z(cbRange, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT;
3270 SUPR3PageFreeEx(pCur, cChunkPages);
3271 }
3272 /*else
3273 {
3274 rc = MMHyperFree(pVM, pCur); - does not work, see the alloc call.
3275 AssertRCReturn(rc, rc);
3276 } */
3277
3278
3279 /* update page count stats */
3280 pVM->pgm.s.cAllPages -= cGuestPages;
3281 pVM->pgm.s.cPrivatePages -= cGuestPages;
3282
3283 /* next */
3284 pCur = pNext;
3285 if (hMmio2 != NIL_PGMMMIO2HANDLE)
3286 {
3287 if (fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3288 break;
3289 hMmio2++;
3290 Assert(pCur->idMmio2 == hMmio2);
3291 Assert(pCur->pDevInsR3 == pDevIns);
3292 Assert(!(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK));
3293 }
3294 }
3295 else
3296 {
3297 pPrev = pCur;
3298 pCur = pCur->pNextR3;
3299 }
3300 }
3301 pgmPhysInvalidatePageMapTLB(pVM);
3302 PGM_UNLOCK(pVM);
3303 return !cFound && hMmio2 != NIL_PGMMMIO2HANDLE ? VERR_NOT_FOUND : rc;
3304}
3305
3306
3307/**
3308 * Maps a MMIO2 region.
3309 *
3310 * This is typically done when a guest / the bios / state loading changes the
3311 * PCI config. The replacing of base memory has the same restrictions as during
3312 * registration, of course.
3313 *
3314 * @returns VBox status code.
3315 *
3316 * @param pVM The cross context VM structure.
3317 * @param pDevIns The device instance owning the region.
3318 * @param hMmio2 The handle of the region to map.
3319 * @param GCPhys The guest-physical address to be remapped.
3320 */
3321VMMR3_INT_DECL(int) PGMR3PhysMmio2Map(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS GCPhys)
3322{
3323 /*
3324 * Validate input.
3325 *
3326 * Note! It's safe to walk the MMIO/MMIO2 list since registrations only
3327 * happens during VM construction.
3328 */
3329 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3330 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3331 AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
3332 AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
3333 AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3334 AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
3335
3336 PPGMREGMMIO2RANGE pFirstMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
3337 AssertReturn(pFirstMmio, VERR_NOT_FOUND);
3338 Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK);
3339
3340 PPGMREGMMIO2RANGE pLastMmio = pFirstMmio;
3341 RTGCPHYS cbRange = 0;
3342 for (;;)
3343 {
3344 AssertReturn(!(pLastMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED), VERR_WRONG_ORDER);
3345 Assert(pLastMmio->RamRange.GCPhys == NIL_RTGCPHYS);
3346 Assert(pLastMmio->RamRange.GCPhysLast == NIL_RTGCPHYS);
3347 Assert(pLastMmio->pDevInsR3 == pFirstMmio->pDevInsR3);
3348 Assert(pLastMmio->iSubDev == pFirstMmio->iSubDev);
3349 Assert(pLastMmio->iRegion == pFirstMmio->iRegion);
3350 cbRange += pLastMmio->RamRange.cb;
3351 if (pLastMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3352 break;
3353 pLastMmio = pLastMmio->pNextR3;
3354 }
3355
3356 RTGCPHYS GCPhysLast = GCPhys + cbRange - 1;
3357 AssertLogRelReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3358
3359 /*
3360 * Find our location in the ram range list, checking for restriction
3361 * we don't bother implementing yet (partially overlapping, multiple
3362 * ram ranges).
3363 */
3364 PGM_LOCK_VOID(pVM);
3365
3366 AssertReturnStmt(!(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED), PGM_UNLOCK(pVM), VERR_WRONG_ORDER);
3367
3368 bool fRamExists = false;
3369 PPGMRAMRANGE pRamPrev = NULL;
3370 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3371 while (pRam && GCPhysLast >= pRam->GCPhys)
3372 {
3373 if ( GCPhys <= pRam->GCPhysLast
3374 && GCPhysLast >= pRam->GCPhys)
3375 {
3376 /* Completely within? */
3377 AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
3378 && GCPhysLast <= pRam->GCPhysLast,
3379 ("%RGp-%RGp (MMIOEx/%s) falls partly outside %RGp-%RGp (%s)\n",
3380 GCPhys, GCPhysLast, pFirstMmio->RamRange.pszDesc,
3381 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3382 PGM_UNLOCK(pVM),
3383 VERR_PGM_RAM_CONFLICT);
3384
3385 /* Check that all the pages are RAM pages. */
3386 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
3387 uint32_t cPagesLeft = cbRange >> GUEST_PAGE_SHIFT;
3388 while (cPagesLeft-- > 0)
3389 {
3390 AssertLogRelMsgReturnStmt(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3391 ("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
3392 GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pFirstMmio->RamRange.pszDesc),
3393 PGM_UNLOCK(pVM),
3394 VERR_PGM_RAM_CONFLICT);
3395 pPage++;
3396 }
3397
3398 /* There can only be one MMIO/MMIO2 chunk matching here! */
3399 AssertLogRelMsgReturnStmt(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK,
3400 ("%RGp-%RGp (MMIOEx/%s, flags %#X) consists of multiple chunks whereas the RAM somehow doesn't!\n",
3401 GCPhys, GCPhysLast, pFirstMmio->RamRange.pszDesc, pFirstMmio->fFlags),
3402 PGM_UNLOCK(pVM),
3403 VERR_PGM_PHYS_MMIO_EX_IPE);
3404
3405 fRamExists = true;
3406 break;
3407 }
3408
3409 /* next */
3410 pRamPrev = pRam;
3411 pRam = pRam->pNextR3;
3412 }
3413 Log(("PGMR3PhysMmio2Map: %RGp-%RGp fRamExists=%RTbool %s\n", GCPhys, GCPhysLast, fRamExists, pFirstMmio->RamRange.pszDesc));
3414
3415
3416 /*
3417 * Make the changes.
3418 */
3419 RTGCPHYS GCPhysCur = GCPhys;
3420 for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3421 {
3422 pCurMmio->RamRange.GCPhys = GCPhysCur;
3423 pCurMmio->RamRange.GCPhysLast = GCPhysCur + pCurMmio->RamRange.cb - 1;
3424 if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3425 {
3426 Assert(pCurMmio->RamRange.GCPhysLast == GCPhysLast);
3427 break;
3428 }
3429 GCPhysCur += pCurMmio->RamRange.cb;
3430 }
3431
3432 if (fRamExists)
3433 {
3434 /*
3435 * Make all the pages in the range MMIO/ZERO pages, freeing any
3436 * RAM pages currently mapped here. This might not be 100% correct
3437 * for PCI memory, but we're doing the same thing for MMIO2 pages.
3438 *
3439 * We replace these MMIO/ZERO pages with real pages in the MMIO2 case.
3440 */
3441 Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK); /* Only one chunk */
3442 Assert(pFirstMmio->pvR3 == pFirstMmio->RamRange.pvR3);
3443 Assert(pFirstMmio->RamRange.pvR3 != NULL);
3444
3445#ifdef VBOX_WITH_PGM_NEM_MODE
3446 /* We cannot mix MMIO2 into a RAM range in simplified memory mode because pRam->pvR3 can't point
3447 both at the RAM and MMIO2, so we won't ever write & read from the actual MMIO2 memory if we try. */
3448 AssertLogRelMsgReturn(!pVM->pgm.s.fNemMode, ("%s at %RGp-%RGp\n", pFirstMmio->RamRange.pszDesc, GCPhys, GCPhysLast),
3449 VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
3450#endif
3451
3452 int rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, pFirstMmio->RamRange.pvR3);
3453 AssertRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
3454
3455 /* Replace the pages, freeing all present RAM pages. */
3456 PPGMPAGE pPageSrc = &pFirstMmio->RamRange.aPages[0];
3457 PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
3458 uint32_t cPagesLeft = pFirstMmio->RamRange.cb >> GUEST_PAGE_SHIFT;
3459 while (cPagesLeft-- > 0)
3460 {
3461 Assert(PGM_PAGE_IS_MMIO(pPageDst));
3462
3463 RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
3464 uint32_t const idPage = PGM_PAGE_GET_PAGEID(pPageSrc);
3465 PGM_PAGE_SET_PAGEID(pVM, pPageDst, idPage);
3466 PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
3467 PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
3468 PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
3469 PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
3470 PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
3471 PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
3472 /* NEM state is set by pgmR3PhysFreePageRange. */
3473
3474 pVM->pgm.s.cZeroPages--;
3475 GCPhys += GUEST_PAGE_SIZE;
3476 pPageSrc++;
3477 pPageDst++;
3478 }
3479
3480 /* Flush physical page map TLB. */
3481 pgmPhysInvalidatePageMapTLB(pVM);
3482
3483 /* Force a PGM pool flush as guest ram references have been changed. */
3484 /** @todo not entirely SMP safe; assuming for now the guest takes care of
3485 * this internally (not touch mapped mmio while changing the mapping). */
3486 PVMCPU pVCpu = VMMGetCpu(pVM);
3487 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
3488 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3489 }
3490 else
3491 {
3492 /*
3493 * No RAM range, insert the ones prepared during registration.
3494 */
3495 for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3496 {
3497#ifdef VBOX_WITH_NATIVE_NEM
3498 /* Tell NEM and get the new NEM state for the pages. */
3499 uint8_t u2NemState = 0;
3500 if (VM_IS_NEM_ENABLED(pVM))
3501 {
3502 int rc = NEMR3NotifyPhysMmioExMapEarly(pVM, pCurMmio->RamRange.GCPhys,
3503 pCurMmio->RamRange.GCPhysLast - pCurMmio->RamRange.GCPhys + 1,
3504 NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2
3505 | (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES
3506 ? NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES : 0),
3507 NULL /*pvRam*/, pCurMmio->RamRange.pvR3,
3508 &u2NemState, &pCurMmio->RamRange.uNemRange);
3509 AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
3510 }
3511#endif
3512
3513 /* Clear the tracking data of pages we're going to reactivate. */
3514 PPGMPAGE pPageSrc = &pCurMmio->RamRange.aPages[0];
3515 uint32_t cPagesLeft = pCurMmio->RamRange.cb >> GUEST_PAGE_SHIFT;
3516 while (cPagesLeft-- > 0)
3517 {
3518 PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
3519 PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
3520#ifdef VBOX_WITH_NATIVE_NEM
3521 PGM_PAGE_SET_NEM_STATE(pPageSrc, u2NemState);
3522#endif
3523 pPageSrc++;
3524 }
3525
3526 /* link in the ram range */
3527 pgmR3PhysLinkRamRange(pVM, &pCurMmio->RamRange, pRamPrev);
3528
3529 if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3530 {
3531 Assert(pCurMmio->RamRange.GCPhysLast == GCPhysLast);
3532 break;
3533 }
3534 pRamPrev = &pCurMmio->RamRange;
3535 }
3536 }
3537
3538 /*
3539 * If the range have dirty page monitoring enabled, enable that.
3540 *
3541 * We ignore failures here for now because if we fail, the whole mapping
3542 * will have to be reversed and we'll end up with nothing at all on the
3543 * screen and a grumpy guest, whereas if we just go on, we'll only have
3544 * visual distortions to gripe about. There will be something in the
3545 * release log.
3546 */
3547 if ( pFirstMmio->pPhysHandlerR3
3548 && (pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
3549 pgmR3PhysMmio2EnableDirtyPageTracing(pVM, pFirstMmio);
3550
3551 /*
3552 * We're good, set the flags and invalid the mapping TLB.
3553 */
3554 for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3555 {
3556 pCurMmio->fFlags |= PGMREGMMIO2RANGE_F_MAPPED;
3557 if (fRamExists)
3558 pCurMmio->fFlags |= PGMREGMMIO2RANGE_F_OVERLAPPING;
3559 else
3560 pCurMmio->fFlags &= ~PGMREGMMIO2RANGE_F_OVERLAPPING;
3561 if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3562 break;
3563 }
3564 pgmPhysInvalidatePageMapTLB(pVM);
3565
3566#ifdef VBOX_WITH_NATIVE_NEM
3567 /*
3568 * Late NEM notification.
3569 */
3570 if (VM_IS_NEM_ENABLED(pVM))
3571 {
3572 int rc;
3573 uint32_t fNemFlags = NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2;
3574 if (pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES)
3575 fNemFlags |= NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES;
3576 if (fRamExists)
3577 rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemFlags | NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
3578 pRam->pvR3 ? (uint8_t *)pRam->pvR3 + GCPhys - pRam->GCPhys : NULL, pFirstMmio->pvR3,
3579 NULL /*puNemRange*/);
3580 else
3581 {
3582 rc = VINF_SUCCESS;
3583 for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3584 {
3585 rc = NEMR3NotifyPhysMmioExMapLate(pVM, pCurMmio->RamRange.GCPhys, pCurMmio->RamRange.cb, fNemFlags,
3586 NULL, pCurMmio->RamRange.pvR3, &pCurMmio->RamRange.uNemRange);
3587 if ((pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK) || RT_FAILURE(rc))
3588 break;
3589 }
3590 }
3591 AssertLogRelRCReturnStmt(rc, PGMR3PhysMmio2Unmap(pVM, pDevIns, hMmio2, GCPhys); PGM_UNLOCK(pVM), rc);
3592 }
3593#endif
3594
3595 PGM_UNLOCK(pVM);
3596
3597 return VINF_SUCCESS;
3598}
3599
3600
3601/**
3602 * Unmaps an MMIO2 region.
3603 *
3604 * This is typically done when a guest / the bios / state loading changes the
3605 * PCI config. The replacing of base memory has the same restrictions as during
3606 * registration, of course.
3607 */
3608VMMR3_INT_DECL(int) PGMR3PhysMmio2Unmap(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS GCPhys)
3609{
3610 /*
3611 * Validate input
3612 */
3613 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3614 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3615 AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
3616 if (GCPhys != NIL_RTGCPHYS)
3617 {
3618 AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
3619 AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3620 }
3621
3622 PPGMREGMMIO2RANGE pFirstMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
3623 AssertReturn(pFirstMmio, VERR_NOT_FOUND);
3624 Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK);
3625
3626 int rc = PGM_LOCK(pVM);
3627 AssertRCReturn(rc, rc);
3628
3629 PPGMREGMMIO2RANGE pLastMmio = pFirstMmio;
3630 RTGCPHYS cbRange = 0;
3631 for (;;)
3632 {
3633 AssertReturnStmt(pLastMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED, PGM_UNLOCK(pVM), VERR_WRONG_ORDER);
3634 AssertReturnStmt(pLastMmio->RamRange.GCPhys == GCPhys + cbRange || GCPhys == NIL_RTGCPHYS, PGM_UNLOCK(pVM), VERR_INVALID_PARAMETER);
3635 Assert(pLastMmio->pDevInsR3 == pFirstMmio->pDevInsR3);
3636 Assert(pLastMmio->iSubDev == pFirstMmio->iSubDev);
3637 Assert(pLastMmio->iRegion == pFirstMmio->iRegion);
3638 cbRange += pLastMmio->RamRange.cb;
3639 if (pLastMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3640 break;
3641 pLastMmio = pLastMmio->pNextR3;
3642 }
3643
3644 Log(("PGMR3PhysMmio2Unmap: %RGp-%RGp %s\n",
3645 pFirstMmio->RamRange.GCPhys, pLastMmio->RamRange.GCPhysLast, pFirstMmio->RamRange.pszDesc));
3646
3647 uint16_t const fOldFlags = pFirstMmio->fFlags;
3648 AssertReturnStmt(fOldFlags & PGMREGMMIO2RANGE_F_MAPPED, PGM_UNLOCK(pVM), VERR_WRONG_ORDER);
3649
3650 /*
3651 * If monitoring dirty pages, we must deregister the handlers first.
3652 */
3653 if ( pFirstMmio->pPhysHandlerR3
3654 && (fOldFlags & PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
3655 pgmR3PhysMmio2DisableDirtyPageTracing(pVM, pFirstMmio);
3656
3657 /*
3658 * Unmap it.
3659 */
3660 int rcRet = VINF_SUCCESS;
3661#ifdef VBOX_WITH_NATIVE_NEM
3662 uint32_t const fNemFlags = NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2
3663 | (fOldFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES
3664 ? NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES : 0);
3665#endif
3666 if (fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING)
3667 {
3668 /*
3669 * We've replaced RAM, replace with zero pages.
3670 *
3671 * Note! This is where we might differ a little from a real system, because
3672 * it's likely to just show the RAM pages as they were before the
3673 * MMIO/MMIO2 region was mapped here.
3674 */
3675 /* Only one chunk allowed when overlapping! */
3676 Assert(fOldFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK);
3677
3678 /* Restore the RAM pages we've replaced. */
3679 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3680 while (pRam->GCPhys > pFirstMmio->RamRange.GCPhysLast)
3681 pRam = pRam->pNextR3;
3682
3683 PPGMPAGE pPageDst = &pRam->aPages[(pFirstMmio->RamRange.GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
3684 uint32_t cPagesLeft = pFirstMmio->RamRange.cb >> GUEST_PAGE_SHIFT;
3685 pVM->pgm.s.cZeroPages += cPagesLeft; /** @todo not correct for NEM mode */
3686
3687#ifdef VBOX_WITH_NATIVE_NEM
3688 if (VM_IS_NEM_ENABLED(pVM)) /* Notify NEM. Note! we cannot be here in simple memory mode, see mapping function. */
3689 {
3690 uint8_t u2State = UINT8_MAX;
3691 rc = NEMR3NotifyPhysMmioExUnmap(pVM, pFirstMmio->RamRange.GCPhys, pFirstMmio->RamRange.cb,
3692 fNemFlags | NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
3693 pRam->pvR3
3694 ? (uint8_t *)pRam->pvR3 + pFirstMmio->RamRange.GCPhys - pRam->GCPhys : NULL,
3695 pFirstMmio->pvR3, &u2State, &pRam->uNemRange);
3696 AssertRCStmt(rc, rcRet = rc);
3697 if (u2State != UINT8_MAX)
3698 pgmPhysSetNemStateForPages(pPageDst, cPagesLeft, u2State);
3699 }
3700#endif
3701
3702 while (cPagesLeft-- > 0)
3703 {
3704 PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
3705 pPageDst++;
3706 }
3707
3708 /* Flush physical page map TLB. */
3709 pgmPhysInvalidatePageMapTLB(pVM);
3710
3711 /* Update range state. */
3712 pFirstMmio->RamRange.GCPhys = NIL_RTGCPHYS;
3713 pFirstMmio->RamRange.GCPhysLast = NIL_RTGCPHYS;
3714 pFirstMmio->fFlags &= ~(PGMREGMMIO2RANGE_F_OVERLAPPING | PGMREGMMIO2RANGE_F_MAPPED);
3715 }
3716 else
3717 {
3718 /*
3719 * Unlink the chunks related to the MMIO/MMIO2 region.
3720 */
3721 for (PPGMREGMMIO2RANGE pCurMmio = pFirstMmio; ; pCurMmio = pCurMmio->pNextR3)
3722 {
3723#ifdef VBOX_WITH_NATIVE_NEM
3724 if (VM_IS_NEM_ENABLED(pVM)) /* Notify NEM. */
3725 {
3726 uint8_t u2State = UINT8_MAX;
3727 rc = NEMR3NotifyPhysMmioExUnmap(pVM, pCurMmio->RamRange.GCPhys, pCurMmio->RamRange.cb, fNemFlags,
3728 NULL, pCurMmio->pvR3, &u2State, &pCurMmio->RamRange.uNemRange);
3729 AssertRCStmt(rc, rcRet = rc);
3730 if (u2State != UINT8_MAX)
3731 pgmPhysSetNemStateForPages(pCurMmio->RamRange.aPages, pCurMmio->RamRange.cb >> GUEST_PAGE_SHIFT, u2State);
3732 }
3733#endif
3734 pgmR3PhysUnlinkRamRange(pVM, &pCurMmio->RamRange);
3735 pCurMmio->RamRange.GCPhys = NIL_RTGCPHYS;
3736 pCurMmio->RamRange.GCPhysLast = NIL_RTGCPHYS;
3737 pCurMmio->fFlags &= ~(PGMREGMMIO2RANGE_F_OVERLAPPING | PGMREGMMIO2RANGE_F_MAPPED);
3738 if (pCurMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3739 break;
3740 }
3741 }
3742
3743 /* Force a PGM pool flush as guest ram references have been changed. */
3744 /** @todo not entirely SMP safe; assuming for now the guest takes care
3745 * of this internally (not touch mapped mmio while changing the
3746 * mapping). */
3747 PVMCPU pVCpu = VMMGetCpu(pVM);
3748 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
3749 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3750
3751 pgmPhysInvalidatePageMapTLB(pVM);
3752 pgmPhysInvalidRamRangeTlbs(pVM);
3753
3754 PGM_UNLOCK(pVM);
3755 return rcRet;
3756}
3757
3758
3759/**
3760 * Reduces the mapping size of a MMIO2 region.
3761 *
3762 * This is mainly for dealing with old saved states after changing the default
3763 * size of a mapping region. See PGMDevHlpMMIOExReduce and
3764 * PDMPCIDEV::pfnRegionLoadChangeHookR3.
3765 *
3766 * The region must not currently be mapped when making this call. The VM state
3767 * must be state restore or VM construction.
3768 *
3769 * @returns VBox status code.
3770 * @param pVM The cross context VM structure.
3771 * @param pDevIns The device instance owning the region.
3772 * @param hMmio2 The handle of the region to reduce.
3773 * @param cbRegion The new mapping size.
3774 */
3775VMMR3_INT_DECL(int) PGMR3PhysMmio2Reduce(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS cbRegion)
3776{
3777 /*
3778 * Validate input
3779 */
3780 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3781 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3782 AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
3783 AssertReturn(cbRegion >= X86_PAGE_SIZE, VERR_INVALID_PARAMETER);
3784 AssertReturn(!(cbRegion & X86_PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
3785 VMSTATE enmVmState = VMR3GetState(pVM);
3786 AssertLogRelMsgReturn( enmVmState == VMSTATE_CREATING
3787 || enmVmState == VMSTATE_LOADING,
3788 ("enmVmState=%d (%s)\n", enmVmState, VMR3GetStateName(enmVmState)),
3789 VERR_VM_INVALID_VM_STATE);
3790
3791 int rc = PGM_LOCK(pVM);
3792 AssertRCReturn(rc, rc);
3793
3794 PPGMREGMMIO2RANGE pFirstMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
3795 if (pFirstMmio)
3796 {
3797 Assert(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK);
3798 if (!(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED))
3799 {
3800 /*
3801 * NOTE! Current implementation does not support multiple ranges.
3802 * Implement when there is a real world need and thus a testcase.
3803 */
3804 AssertLogRelMsgStmt(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK,
3805 ("%s: %#x\n", pFirstMmio->RamRange.pszDesc, pFirstMmio->fFlags),
3806 rc = VERR_NOT_SUPPORTED);
3807 if (RT_SUCCESS(rc))
3808 {
3809 /*
3810 * Make the change.
3811 */
3812 Log(("PGMR3PhysMmio2Reduce: %s changes from %RGp bytes (%RGp) to %RGp bytes.\n",
3813 pFirstMmio->RamRange.pszDesc, pFirstMmio->RamRange.cb, pFirstMmio->cbReal, cbRegion));
3814
3815 AssertLogRelMsgStmt(cbRegion <= pFirstMmio->cbReal,
3816 ("%s: cbRegion=%#RGp cbReal=%#RGp\n", pFirstMmio->RamRange.pszDesc, cbRegion, pFirstMmio->cbReal),
3817 rc = VERR_OUT_OF_RANGE);
3818 if (RT_SUCCESS(rc))
3819 {
3820 pFirstMmio->RamRange.cb = cbRegion;
3821 }
3822 }
3823 }
3824 else
3825 rc = VERR_WRONG_ORDER;
3826 }
3827 else
3828 rc = VERR_NOT_FOUND;
3829
3830 PGM_UNLOCK(pVM);
3831 return rc;
3832}
3833
3834
3835/**
3836 * Validates @a hMmio2, making sure it belongs to @a pDevIns.
3837 *
3838 * @returns VBox status code.
3839 * @param pVM The cross context VM structure.
3840 * @param pDevIns The device which allegedly owns @a hMmio2.
3841 * @param hMmio2 The handle to validate.
3842 */
3843VMMR3_INT_DECL(int) PGMR3PhysMmio2ValidateHandle(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
3844{
3845 /*
3846 * Validate input
3847 */
3848 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3849 AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
3850
3851 /*
3852 * Just do this the simple way. No need for locking as this is only taken at
3853 */
3854 PGM_LOCK_VOID(pVM);
3855 PPGMREGMMIO2RANGE pFirstMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
3856 PGM_UNLOCK(pVM);
3857 AssertReturn(pFirstMmio, VERR_INVALID_HANDLE);
3858 AssertReturn(pFirstMmio->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, VERR_INVALID_HANDLE);
3859 return VINF_SUCCESS;
3860}
3861
3862
3863/**
3864 * Gets the mapping address of an MMIO2 region.
3865 *
3866 * @returns Mapping address, NIL_RTGCPHYS if not mapped or invalid handle.
3867 *
3868 * @param pVM The cross context VM structure.
3869 * @param pDevIns The device owning the MMIO2 handle.
3870 * @param hMmio2 The region handle.
3871 */
3872VMMR3_INT_DECL(RTGCPHYS) PGMR3PhysMmio2GetMappingAddress(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
3873{
3874 AssertPtrReturn(pDevIns, NIL_RTGCPHYS);
3875
3876 PPGMREGMMIO2RANGE pFirstRegMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
3877 AssertReturn(pFirstRegMmio, NIL_RTGCPHYS);
3878
3879 if (pFirstRegMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
3880 return pFirstRegMmio->RamRange.GCPhys;
3881 return NIL_RTGCPHYS;
3882}
3883
3884
3885/**
3886 * Worker for PGMR3PhysMmio2QueryAndResetDirtyBitmap.
3887 *
3888 * Called holding the PGM lock.
3889 */
3890static int pgmR3PhysMmio2QueryAndResetDirtyBitmapLocked(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2,
3891 void *pvBitmap, size_t cbBitmap)
3892{
3893 /*
3894 * Continue validation.
3895 */
3896 PPGMREGMMIO2RANGE pFirstRegMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
3897 AssertReturn(pFirstRegMmio, VERR_INVALID_HANDLE);
3898 AssertReturn( (pFirstRegMmio->fFlags & (PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES | PGMREGMMIO2RANGE_F_FIRST_CHUNK))
3899 == (PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES | PGMREGMMIO2RANGE_F_FIRST_CHUNK),
3900 VERR_INVALID_FUNCTION);
3901 AssertReturn(pDevIns == pFirstRegMmio->pDevInsR3, VERR_NOT_OWNER);
3902
3903 RTGCPHYS cbTotal = 0;
3904 uint16_t fTotalDirty = 0;
3905 for (PPGMREGMMIO2RANGE pCur = pFirstRegMmio;;)
3906 {
3907 cbTotal += pCur->RamRange.cb; /* Not using cbReal here, because NEM is not in on the creating, only the mapping. */
3908 fTotalDirty |= pCur->fFlags;
3909 if (pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3910 break;
3911 pCur = pCur->pNextR3;
3912 AssertPtrReturn(pCur, VERR_INTERNAL_ERROR_5);
3913 AssertReturn( (pCur->fFlags & (PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES | PGMREGMMIO2RANGE_F_FIRST_CHUNK))
3914 == PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES,
3915 VERR_INTERNAL_ERROR_4);
3916 }
3917 size_t const cbTotalBitmap = RT_ALIGN_T(cbTotal, GUEST_PAGE_SIZE * 64, RTGCPHYS) / GUEST_PAGE_SIZE / 8;
3918
3919 if (cbBitmap)
3920 {
3921 AssertPtrReturn(pvBitmap, VERR_INVALID_POINTER);
3922 AssertReturn(RT_ALIGN_P(pvBitmap, sizeof(uint64_t)) == pvBitmap, VERR_INVALID_POINTER);
3923 AssertReturn(cbBitmap == cbTotalBitmap, VERR_INVALID_PARAMETER);
3924 }
3925
3926 /*
3927 * Do the work.
3928 */
3929 int rc = VINF_SUCCESS;
3930 if (pvBitmap)
3931 {
3932#ifdef VBOX_WITH_PGM_NEM_MODE
3933 if (pFirstRegMmio->pPhysHandlerR3 == NULL)
3934 {
3935/** @todo This does not integrate at all with --execute-all-in-iem, leaving the
3936 * screen blank when using it together with --driverless. Fixing this won't be
3937 * entirely easy as we take the PGM_PAGE_HNDL_PHYS_STATE_DISABLED page status to
3938 * mean a dirty page. */
3939 AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
3940 uint8_t *pbBitmap = (uint8_t *)pvBitmap;
3941 for (PPGMREGMMIO2RANGE pCur = pFirstRegMmio; pCur; pCur = pCur->pNextR3)
3942 {
3943 size_t const cbBitmapChunk = pCur->RamRange.cb / GUEST_PAGE_SIZE / 8;
3944 Assert((RTGCPHYS)cbBitmapChunk * GUEST_PAGE_SIZE * 8 == pCur->RamRange.cb);
3945 int rc2 = NEMR3PhysMmio2QueryAndResetDirtyBitmap(pVM, pCur->RamRange.GCPhys, pCur->RamRange.cb,
3946 pCur->RamRange.uNemRange, pbBitmap, cbBitmapChunk);
3947 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3948 rc = rc2;
3949 if (pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3950 break;
3951 pbBitmap += pCur->RamRange.cb / GUEST_PAGE_SIZE / 8;
3952 }
3953 }
3954 else
3955#endif
3956 if (fTotalDirty & PGMREGMMIO2RANGE_F_IS_DIRTY)
3957 {
3958 if ( (pFirstRegMmio->fFlags & (PGMREGMMIO2RANGE_F_MAPPED | PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
3959 == (PGMREGMMIO2RANGE_F_MAPPED | PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
3960 {
3961 /*
3962 * Reset each chunk, gathering dirty bits.
3963 */
3964 RT_BZERO(pvBitmap, cbBitmap); /* simpler for now. */
3965 uint32_t iPageNo = 0;
3966 for (PPGMREGMMIO2RANGE pCur = pFirstRegMmio; pCur; pCur = pCur->pNextR3)
3967 {
3968 if (pCur->fFlags & PGMREGMMIO2RANGE_F_IS_DIRTY)
3969 {
3970 int rc2 = pgmHandlerPhysicalResetMmio2WithBitmap(pVM, pCur->RamRange.GCPhys, pvBitmap, iPageNo);
3971 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3972 rc = rc2;
3973 pCur->fFlags &= ~PGMREGMMIO2RANGE_F_IS_DIRTY;
3974 }
3975 if (pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3976 break;
3977 iPageNo += pCur->RamRange.cb >> GUEST_PAGE_SHIFT;
3978 }
3979 }
3980 else
3981 {
3982 /*
3983 * If not mapped or tracking is disabled, we return the
3984 * PGMREGMMIO2RANGE_F_IS_DIRTY status for all pages. We cannot
3985 * get more accurate data than that after unmapping or disabling.
3986 */
3987 RT_BZERO(pvBitmap, cbBitmap);
3988 uint32_t iPageNo = 0;
3989 for (PPGMREGMMIO2RANGE pCur = pFirstRegMmio; pCur; pCur = pCur->pNextR3)
3990 {
3991 if (pCur->fFlags & PGMREGMMIO2RANGE_F_IS_DIRTY)
3992 {
3993 ASMBitSetRange(pvBitmap, iPageNo, iPageNo + (pCur->RamRange.cb >> GUEST_PAGE_SHIFT));
3994 pCur->fFlags &= ~PGMREGMMIO2RANGE_F_IS_DIRTY;
3995 }
3996 if (pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3997 break;
3998 iPageNo += pCur->RamRange.cb >> GUEST_PAGE_SHIFT;
3999 }
4000 }
4001 }
4002 /*
4003 * No dirty chunks.
4004 */
4005 else
4006 RT_BZERO(pvBitmap, cbBitmap);
4007 }
4008 /*
4009 * No bitmap. Reset the region if tracking is currently enabled.
4010 */
4011 else if ( (pFirstRegMmio->fFlags & (PGMREGMMIO2RANGE_F_MAPPED | PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4012 == (PGMREGMMIO2RANGE_F_MAPPED | PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4013 {
4014#ifdef VBOX_WITH_PGM_NEM_MODE
4015 if (pFirstRegMmio->pPhysHandlerR3 == NULL)
4016 {
4017 AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
4018 for (PPGMREGMMIO2RANGE pCur = pFirstRegMmio; pCur; pCur = pCur->pNextR3)
4019 {
4020 int rc2 = NEMR3PhysMmio2QueryAndResetDirtyBitmap(pVM, pCur->RamRange.GCPhys, pCur->RamRange.cb,
4021 pCur->RamRange.uNemRange, NULL, 0);
4022 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4023 rc = rc2;
4024 if (pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
4025 break;
4026 }
4027 }
4028 else
4029#endif
4030 {
4031 for (PPGMREGMMIO2RANGE pCur = pFirstRegMmio; pCur; pCur = pCur->pNextR3)
4032 {
4033 pCur->fFlags &= ~PGMREGMMIO2RANGE_F_IS_DIRTY;
4034 int rc2 = PGMHandlerPhysicalReset(pVM, pCur->RamRange.GCPhys);
4035 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4036 rc = rc2;
4037 if (pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
4038 break;
4039 }
4040 }
4041 }
4042
4043 return rc;
4044}
4045
4046
4047/**
4048 * Queries the dirty page bitmap and resets the monitoring.
4049 *
4050 * The PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES flag must be specified when
4051 * creating the range for this to work.
4052 *
4053 * @returns VBox status code.
4054 * @retval VERR_INVALID_FUNCTION if not created using
4055 * PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES.
4056 * @param pVM The cross context VM structure.
4057 * @param pDevIns The device owning the MMIO2 handle.
4058 * @param hMmio2 The region handle.
4059 * @param pvBitmap The output bitmap. Must be 8-byte aligned. Ignored
4060 * when @a cbBitmap is zero.
4061 * @param cbBitmap The size of the bitmap. Must be the size of the whole
4062 * MMIO2 range, rounded up to the nearest 8 bytes.
4063 * When zero only a reset is done.
4064 */
4065VMMR3_INT_DECL(int) PGMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2,
4066 void *pvBitmap, size_t cbBitmap)
4067{
4068 /*
4069 * Do some basic validation before grapping the PGM lock and continuing.
4070 */
4071 AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
4072 AssertReturn(RT_ALIGN_Z(cbBitmap, sizeof(uint64_t)) == cbBitmap, VERR_INVALID_PARAMETER);
4073 int rc = PGM_LOCK(pVM);
4074 if (RT_SUCCESS(rc))
4075 {
4076 STAM_PROFILE_START(&pVM->pgm.s.StatMmio2QueryAndResetDirtyBitmap, a);
4077 rc = pgmR3PhysMmio2QueryAndResetDirtyBitmapLocked(pVM, pDevIns, hMmio2, pvBitmap, cbBitmap);
4078 STAM_PROFILE_STOP(&pVM->pgm.s.StatMmio2QueryAndResetDirtyBitmap, a);
4079 PGM_UNLOCK(pVM);
4080 }
4081 return rc;
4082}
4083
4084
4085/**
4086 * Worker for PGMR3PhysMmio2ControlDirtyPageTracking
4087 *
4088 * Called owning the PGM lock.
4089 */
4090static int pgmR3PhysMmio2ControlDirtyPageTrackingLocked(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, bool fEnabled)
4091{
4092 /*
4093 * Continue validation.
4094 */
4095 PPGMREGMMIO2RANGE pFirstRegMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
4096 AssertReturn(pFirstRegMmio, VERR_INVALID_HANDLE);
4097 AssertReturn( (pFirstRegMmio->fFlags & (PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES | PGMREGMMIO2RANGE_F_FIRST_CHUNK))
4098 == (PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES | PGMREGMMIO2RANGE_F_FIRST_CHUNK)
4099 , VERR_INVALID_FUNCTION);
4100 AssertReturn(pDevIns == pFirstRegMmio->pDevInsR3, VERR_NOT_OWNER);
4101
4102#ifdef VBOX_WITH_PGM_NEM_MODE
4103 /*
4104 * This is a nop if NEM is responsible for doing the tracking, we simply
4105 * leave the tracking on all the time there.
4106 */
4107 if (pFirstRegMmio->pPhysHandlerR3 == NULL)
4108 {
4109 AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
4110 return VINF_SUCCESS;
4111 }
4112#endif
4113
4114 /*
4115 * Anyting needing doing?
4116 */
4117 if (fEnabled != RT_BOOL(pFirstRegMmio->fFlags & PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4118 {
4119 LogFlowFunc(("fEnabled=%RTbool %s\n", fEnabled, pFirstRegMmio->RamRange.pszDesc));
4120
4121 /*
4122 * Update the PGMREGMMIO2RANGE_F_TRACKING_ENABLED flag.
4123 */
4124 for (PPGMREGMMIO2RANGE pCur = pFirstRegMmio;;)
4125 {
4126 if (fEnabled)
4127 pCur->fFlags |= PGMREGMMIO2RANGE_F_TRACKING_ENABLED;
4128 else
4129 pCur->fFlags &= ~PGMREGMMIO2RANGE_F_TRACKING_ENABLED;
4130 if (pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
4131 break;
4132 pCur = pCur->pNextR3;
4133 AssertPtrReturn(pCur, VERR_INTERNAL_ERROR_5);
4134 AssertReturn( (pCur->fFlags & (PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES | PGMREGMMIO2RANGE_F_FIRST_CHUNK))
4135 == PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES
4136 , VERR_INTERNAL_ERROR_4);
4137 }
4138
4139 /*
4140 * Enable/disable handlers if currently mapped.
4141 *
4142 * We ignore status codes here as we've already changed the flags and
4143 * returning a failure status now would be confusing. Besides, the two
4144 * functions will continue past failures. As argued in the mapping code,
4145 * it's in the release log.
4146 */
4147 if (pFirstRegMmio->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
4148 {
4149 if (fEnabled)
4150 pgmR3PhysMmio2EnableDirtyPageTracing(pVM, pFirstRegMmio);
4151 else
4152 pgmR3PhysMmio2DisableDirtyPageTracing(pVM, pFirstRegMmio);
4153 }
4154 }
4155 else
4156 LogFlowFunc(("fEnabled=%RTbool %s - no change\n", fEnabled, pFirstRegMmio->RamRange.pszDesc));
4157
4158 return VINF_SUCCESS;
4159}
4160
4161
4162/**
4163 * Controls the dirty page tracking for an MMIO2 range.
4164 *
4165 * @returns VBox status code.
4166 * @param pVM The cross context VM structure.
4167 * @param pDevIns The device owning the MMIO2 memory.
4168 * @param hMmio2 The handle of the region.
4169 * @param fEnabled The new tracking state.
4170 */
4171VMMR3_INT_DECL(int) PGMR3PhysMmio2ControlDirtyPageTracking(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, bool fEnabled)
4172{
4173 /*
4174 * Do some basic validation before grapping the PGM lock and continuing.
4175 */
4176 AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
4177 int rc = PGM_LOCK(pVM);
4178 if (RT_SUCCESS(rc))
4179 {
4180 rc = pgmR3PhysMmio2ControlDirtyPageTrackingLocked(pVM, pDevIns, hMmio2, fEnabled);
4181 PGM_UNLOCK(pVM);
4182 }
4183 return rc;
4184}
4185
4186
4187/**
4188 * Changes the region number of an MMIO2 region.
4189 *
4190 * This is only for dealing with save state issues, nothing else.
4191 *
4192 * @return VBox status code.
4193 *
4194 * @param pVM The cross context VM structure.
4195 * @param pDevIns The device owning the MMIO2 memory.
4196 * @param hMmio2 The handle of the region.
4197 * @param iNewRegion The new region index.
4198 *
4199 * @thread EMT(0)
4200 * @sa @bugref{9359}
4201 */
4202VMMR3_INT_DECL(int) PGMR3PhysMmio2ChangeRegionNo(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, uint32_t iNewRegion)
4203{
4204 /*
4205 * Validate input.
4206 */
4207 VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
4208 VM_ASSERT_STATE_RETURN(pVM, VMSTATE_LOADING, VERR_VM_INVALID_VM_STATE);
4209 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
4210 AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
4211 AssertReturn(iNewRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
4212
4213 AssertReturn(pVM->enmVMState == VMSTATE_LOADING, VERR_INVALID_STATE);
4214
4215 int rc = PGM_LOCK(pVM);
4216 AssertRCReturn(rc, rc);
4217
4218 PPGMREGMMIO2RANGE pFirstRegMmio = pgmR3PhysMmio2Find(pVM, pDevIns, UINT32_MAX, UINT32_MAX, hMmio2);
4219 AssertReturnStmt(pFirstRegMmio, PGM_UNLOCK(pVM), VERR_NOT_FOUND);
4220 AssertReturnStmt(pgmR3PhysMmio2Find(pVM, pDevIns, pFirstRegMmio->iSubDev, iNewRegion, NIL_PGMMMIO2HANDLE) == NULL,
4221 PGM_UNLOCK(pVM), VERR_RESOURCE_IN_USE);
4222
4223 /*
4224 * Make the change.
4225 */
4226 pFirstRegMmio->iRegion = (uint8_t)iNewRegion;
4227
4228 PGM_UNLOCK(pVM);
4229 return VINF_SUCCESS;
4230}
4231
4232
4233
4234/*********************************************************************************************************************************
4235* ROM *
4236*********************************************************************************************************************************/
4237
4238/**
4239 * Worker for PGMR3PhysRomRegister.
4240 *
4241 * This is here to simplify lock management, i.e. the caller does all the
4242 * locking and we can simply return without needing to remember to unlock
4243 * anything first.
4244 *
4245 * @returns VBox status code.
4246 * @param pVM The cross context VM structure.
4247 * @param pDevIns The device instance owning the ROM.
4248 * @param GCPhys First physical address in the range.
4249 * Must be page aligned!
4250 * @param cb The size of the range (in bytes).
4251 * Must be page aligned!
4252 * @param pvBinary Pointer to the binary data backing the ROM image.
4253 * @param cbBinary The size of the binary data pvBinary points to.
4254 * This must be less or equal to @a cb.
4255 * @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
4256 * and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
4257 * @param pszDesc Pointer to description string. This must not be freed.
4258 */
4259static int pgmR3PhysRomRegisterLocked(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
4260 const void *pvBinary, uint32_t cbBinary, uint8_t fFlags, const char *pszDesc)
4261{
4262 /*
4263 * Validate input.
4264 */
4265 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
4266 AssertReturn(RT_ALIGN_T(GCPhys, GUEST_PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
4267 AssertReturn(RT_ALIGN_T(cb, GUEST_PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
4268 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
4269 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
4270 AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
4271 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
4272 AssertReturn(!(fFlags & ~PGMPHYS_ROM_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
4273 VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
4274
4275 const uint32_t cGuestPages = cb >> GUEST_PAGE_SHIFT;
4276#ifdef VBOX_WITH_PGM_NEM_MODE
4277 const uint32_t cHostPages = RT_ALIGN_T(cb, HOST_PAGE_SIZE, RTGCPHYS) >> HOST_PAGE_SHIFT;
4278#endif
4279
4280 /*
4281 * Find the ROM location in the ROM list first.
4282 */
4283 PPGMROMRANGE pRomPrev = NULL;
4284 PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
4285 while (pRom && GCPhysLast >= pRom->GCPhys)
4286 {
4287 if ( GCPhys <= pRom->GCPhysLast
4288 && GCPhysLast >= pRom->GCPhys)
4289 AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
4290 GCPhys, GCPhysLast, pszDesc,
4291 pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
4292 VERR_PGM_RAM_CONFLICT);
4293 /* next */
4294 pRomPrev = pRom;
4295 pRom = pRom->pNextR3;
4296 }
4297
4298 /*
4299 * Find the RAM location and check for conflicts.
4300 *
4301 * Conflict detection is a bit different than for RAM registration since a
4302 * ROM can be located within a RAM range. So, what we have to check for is
4303 * other memory types (other than RAM that is) and that we don't span more
4304 * than one RAM range (lazy).
4305 */
4306 bool fRamExists = false;
4307 PPGMRAMRANGE pRamPrev = NULL;
4308 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4309 while (pRam && GCPhysLast >= pRam->GCPhys)
4310 {
4311 if ( GCPhys <= pRam->GCPhysLast
4312 && GCPhysLast >= pRam->GCPhys)
4313 {
4314 /* completely within? */
4315 AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
4316 && GCPhysLast <= pRam->GCPhysLast,
4317 ("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
4318 GCPhys, GCPhysLast, pszDesc,
4319 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
4320 VERR_PGM_RAM_CONFLICT);
4321 fRamExists = true;
4322 break;
4323 }
4324
4325 /* next */
4326 pRamPrev = pRam;
4327 pRam = pRam->pNextR3;
4328 }
4329 if (fRamExists)
4330 {
4331 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
4332 uint32_t cPagesLeft = cGuestPages;
4333 while (cPagesLeft-- > 0)
4334 {
4335 AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
4336 ("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
4337 pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT),
4338 pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
4339 Assert(PGM_PAGE_IS_ZERO(pPage) || PGM_IS_IN_NEM_MODE(pVM));
4340 pPage++;
4341 }
4342 }
4343
4344 /*
4345 * Update the base memory reservation if necessary.
4346 */
4347 uint32_t cExtraBaseCost = fRamExists ? 0 : cGuestPages;
4348 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4349 cExtraBaseCost += cGuestPages;
4350 if (cExtraBaseCost)
4351 {
4352 int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
4353 if (RT_FAILURE(rc))
4354 return rc;
4355 }
4356
4357#ifdef VBOX_WITH_NATIVE_NEM
4358 /*
4359 * Early NEM notification before we've made any changes or anything.
4360 */
4361 uint32_t const fNemNotify = (fRamExists ? NEM_NOTIFY_PHYS_ROM_F_REPLACE : 0)
4362 | (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED ? NEM_NOTIFY_PHYS_ROM_F_SHADOW : 0);
4363 uint8_t u2NemState = UINT8_MAX;
4364 uint32_t uNemRange = 0;
4365 if (VM_IS_NEM_ENABLED(pVM))
4366 {
4367 int rc = NEMR3NotifyPhysRomRegisterEarly(pVM, GCPhys, cGuestPages << GUEST_PAGE_SHIFT,
4368 fRamExists ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhys) : NULL,
4369 fNemNotify, &u2NemState, fRamExists ? &pRam->uNemRange : &uNemRange);
4370 AssertLogRelRCReturn(rc, rc);
4371 }
4372#endif
4373
4374 /*
4375 * Allocate memory for the virgin copy of the RAM. In simplified memory mode,
4376 * we allocate memory for any ad-hoc RAM range and for shadow pages.
4377 */
4378 PGMMALLOCATEPAGESREQ pReq = NULL;
4379#ifdef VBOX_WITH_PGM_NEM_MODE
4380 void *pvRam = NULL;
4381 void *pvAlt = NULL;
4382 if (pVM->pgm.s.fNemMode)
4383 {
4384 if (!fRamExists)
4385 {
4386 int rc = SUPR3PageAlloc(cHostPages, 0, &pvRam);
4387 if (RT_FAILURE(rc))
4388 return rc;
4389 }
4390 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4391 {
4392 int rc = SUPR3PageAlloc(cHostPages, 0, &pvAlt);
4393 if (RT_FAILURE(rc))
4394 {
4395 if (pvRam)
4396 SUPR3PageFree(pvRam, cHostPages);
4397 return rc;
4398 }
4399 }
4400 }
4401 else
4402#endif
4403 {
4404 int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cGuestPages, GMMACCOUNT_BASE);
4405 AssertRCReturn(rc, rc);
4406
4407 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++)
4408 {
4409 pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << GUEST_PAGE_SHIFT);
4410 pReq->aPages[iPage].fZeroed = false;
4411 pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
4412 pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
4413 }
4414
4415 rc = GMMR3AllocatePagesPerform(pVM, pReq);
4416 if (RT_FAILURE(rc))
4417 {
4418 GMMR3AllocatePagesCleanup(pReq);
4419 return rc;
4420 }
4421 }
4422
4423 /*
4424 * Allocate the new ROM range and RAM range (if necessary).
4425 */
4426 PPGMROMRANGE pRomNew = NULL;
4427 RTR0PTR pRomNewR0 = NIL_RTR0PTR;
4428 size_t const cbRomRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMROMRANGE, aPages[cGuestPages]), 128);
4429 size_t const cbRamRange = fRamExists ? 0 : RT_UOFFSETOF_DYN(PGMROMRANGE, aPages[cGuestPages]);
4430 size_t const cRangePages = RT_ALIGN_Z(cbRomRange + cbRamRange, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT;
4431 int rc = SUPR3PageAllocEx(cRangePages, 0 /*fFlags*/, (void **)&pRomNew, &pRomNewR0, NULL /*paPages*/);
4432 if (RT_SUCCESS(rc))
4433 {
4434
4435 /*
4436 * Initialize and insert the RAM range (if required).
4437 */
4438 PPGMRAMRANGE pRamNew;
4439 uint32_t const idxFirstRamPage = fRamExists ? (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT : 0;
4440 PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
4441 if (!fRamExists)
4442 {
4443 /* New RAM range. */
4444 pRamNew = (PPGMRAMRANGE)((uintptr_t)pRomNew + cbRomRange);
4445 pRamNew->pSelfR0 = !pRomNewR0 ? NIL_RTR0PTR : pRomNewR0 + cbRomRange;
4446 pRamNew->GCPhys = GCPhys;
4447 pRamNew->GCPhysLast = GCPhysLast;
4448 pRamNew->cb = cb;
4449 pRamNew->pszDesc = pszDesc;
4450 pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
4451 pRamNew->pvR3 = NULL;
4452 pRamNew->paLSPages = NULL;
4453#ifdef VBOX_WITH_NATIVE_NEM
4454 pRamNew->uNemRange = uNemRange;
4455#endif
4456
4457 PPGMPAGE pRamPage = &pRamNew->aPages[idxFirstRamPage];
4458#ifdef VBOX_WITH_PGM_NEM_MODE
4459 if (pVM->pgm.s.fNemMode)
4460 {
4461 AssertPtr(pvRam); Assert(pReq == NULL);
4462 pRamNew->pvR3 = pvRam;
4463 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4464 {
4465 PGM_PAGE_INIT(pRamPage, UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
4466 PGMPAGETYPE_ROM, PGM_PAGE_STATE_ALLOCATED);
4467 pRomPage->Virgin = *pRamPage;
4468 }
4469 }
4470 else
4471#endif
4472 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4473 {
4474 PGM_PAGE_INIT(pRamPage,
4475 pReq->aPages[iPage].HCPhysGCPhys,
4476 pReq->aPages[iPage].idPage,
4477 PGMPAGETYPE_ROM,
4478 PGM_PAGE_STATE_ALLOCATED);
4479
4480 pRomPage->Virgin = *pRamPage;
4481 }
4482
4483 pVM->pgm.s.cAllPages += cGuestPages;
4484 pVM->pgm.s.cPrivatePages += cGuestPages;
4485 pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
4486 }
4487 else
4488 {
4489 /* Existing RAM range. */
4490 PPGMPAGE pRamPage = &pRam->aPages[idxFirstRamPage];
4491#ifdef VBOX_WITH_PGM_NEM_MODE
4492 if (pVM->pgm.s.fNemMode)
4493 {
4494 Assert(pvRam == NULL); Assert(pReq == NULL);
4495 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4496 {
4497 Assert(PGM_PAGE_GET_HCPHYS(pRamPage) == UINT64_C(0x0000fffffffff000));
4498 Assert(PGM_PAGE_GET_PAGEID(pRamPage) == NIL_GMM_PAGEID);
4499 Assert(PGM_PAGE_GET_STATE(pRamPage) == PGM_PAGE_STATE_ALLOCATED);
4500 PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_ROM);
4501 PGM_PAGE_SET_STATE(pVM, pRamPage, PGM_PAGE_STATE_ALLOCATED);
4502 PGM_PAGE_SET_PDE_TYPE(pVM, pRamPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4503 PGM_PAGE_SET_PTE_INDEX(pVM, pRamPage, 0);
4504 PGM_PAGE_SET_TRACKING(pVM, pRamPage, 0);
4505
4506 pRomPage->Virgin = *pRamPage;
4507 }
4508 }
4509 else
4510#endif
4511 {
4512 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4513 {
4514 PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_ROM);
4515 PGM_PAGE_SET_HCPHYS(pVM, pRamPage, pReq->aPages[iPage].HCPhysGCPhys);
4516 PGM_PAGE_SET_STATE(pVM, pRamPage, PGM_PAGE_STATE_ALLOCATED);
4517 PGM_PAGE_SET_PAGEID(pVM, pRamPage, pReq->aPages[iPage].idPage);
4518 PGM_PAGE_SET_PDE_TYPE(pVM, pRamPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4519 PGM_PAGE_SET_PTE_INDEX(pVM, pRamPage, 0);
4520 PGM_PAGE_SET_TRACKING(pVM, pRamPage, 0);
4521
4522 pRomPage->Virgin = *pRamPage;
4523 }
4524 pVM->pgm.s.cZeroPages -= cGuestPages;
4525 pVM->pgm.s.cPrivatePages += cGuestPages;
4526 }
4527 pRamNew = pRam;
4528 }
4529
4530#ifdef VBOX_WITH_NATIVE_NEM
4531 /* Set the NEM state of the pages if needed. */
4532 if (u2NemState != UINT8_MAX)
4533 pgmPhysSetNemStateForPages(&pRamNew->aPages[idxFirstRamPage], cGuestPages, u2NemState);
4534#endif
4535
4536 /* Flush physical page map TLB. */
4537 pgmPhysInvalidatePageMapTLB(pVM);
4538
4539 /*
4540 * Register the ROM access handler.
4541 */
4542 rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType, GCPhys, pszDesc);
4543 if (RT_SUCCESS(rc))
4544 {
4545 /*
4546 * Copy the image over to the virgin pages.
4547 * This must be done after linking in the RAM range.
4548 */
4549 size_t cbBinaryLeft = cbBinary;
4550 PPGMPAGE pRamPage = &pRamNew->aPages[idxFirstRamPage];
4551 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++)
4552 {
4553 void *pvDstPage;
4554 rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << GUEST_PAGE_SHIFT), &pvDstPage);
4555 if (RT_FAILURE(rc))
4556 {
4557 VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
4558 break;
4559 }
4560 if (cbBinaryLeft >= GUEST_PAGE_SIZE)
4561 {
4562 memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << GUEST_PAGE_SHIFT), GUEST_PAGE_SIZE);
4563 cbBinaryLeft -= GUEST_PAGE_SIZE;
4564 }
4565 else
4566 {
4567 RT_BZERO(pvDstPage, GUEST_PAGE_SIZE); /* (shouldn't be necessary, but can't hurt either) */
4568 if (cbBinaryLeft > 0)
4569 {
4570 memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << GUEST_PAGE_SHIFT), cbBinaryLeft);
4571 cbBinaryLeft = 0;
4572 }
4573 }
4574 }
4575 if (RT_SUCCESS(rc))
4576 {
4577 /*
4578 * Initialize the ROM range.
4579 * Note that the Virgin member of the pages has already been initialized above.
4580 */
4581 pRomNew->pSelfR0 = pRomNewR0;
4582 pRomNew->GCPhys = GCPhys;
4583 pRomNew->GCPhysLast = GCPhysLast;
4584 pRomNew->cb = cb;
4585 pRomNew->fFlags = fFlags;
4586 pRomNew->idSavedState = UINT8_MAX;
4587 pRomNew->cbOriginal = cbBinary;
4588 pRomNew->pszDesc = pszDesc;
4589#ifdef VBOX_WITH_PGM_NEM_MODE
4590 pRomNew->pbR3Alternate = (uint8_t *)pvAlt;
4591#endif
4592 pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
4593 ? pvBinary : RTMemDup(pvBinary, cbBinary);
4594 if (pRomNew->pvOriginal)
4595 {
4596 for (unsigned iPage = 0; iPage < cGuestPages; iPage++)
4597 {
4598 PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
4599 pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
4600#ifdef VBOX_WITH_PGM_NEM_MODE
4601 if (pVM->pgm.s.fNemMode)
4602 PGM_PAGE_INIT(&pPage->Shadow, UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
4603 PGMPAGETYPE_ROM_SHADOW, PGM_PAGE_STATE_ALLOCATED);
4604 else
4605#endif
4606 PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
4607 }
4608
4609 /* update the page count stats for the shadow pages. */
4610 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4611 {
4612#ifdef VBOX_WITH_PGM_NEM_MODE
4613 if (pVM->pgm.s.fNemMode)
4614 pVM->pgm.s.cPrivatePages += cGuestPages;
4615 else
4616#endif
4617 pVM->pgm.s.cZeroPages += cGuestPages;
4618 pVM->pgm.s.cAllPages += cGuestPages;
4619 }
4620
4621 /*
4622 * Insert the ROM range, tell REM and return successfully.
4623 */
4624 pRomNew->pNextR3 = pRom;
4625 pRomNew->pNextR0 = pRom ? pRom->pSelfR0 : NIL_RTR0PTR;
4626
4627 if (pRomPrev)
4628 {
4629 pRomPrev->pNextR3 = pRomNew;
4630 pRomPrev->pNextR0 = pRomNew->pSelfR0;
4631 }
4632 else
4633 {
4634 pVM->pgm.s.pRomRangesR3 = pRomNew;
4635 pVM->pgm.s.pRomRangesR0 = pRomNew->pSelfR0;
4636 }
4637
4638 pgmPhysInvalidatePageMapTLB(pVM);
4639#ifdef VBOX_WITH_PGM_NEM_MODE
4640 if (!pVM->pgm.s.fNemMode)
4641#endif
4642 GMMR3AllocatePagesCleanup(pReq);
4643
4644#ifdef VBOX_WITH_NATIVE_NEM
4645 /*
4646 * Notify NEM again.
4647 */
4648 if (VM_IS_NEM_ENABLED(pVM))
4649 {
4650 u2NemState = UINT8_MAX;
4651 rc = NEMR3NotifyPhysRomRegisterLate(pVM, GCPhys, cb, PGM_RAMRANGE_CALC_PAGE_R3PTR(pRamNew, GCPhys),
4652 fNemNotify, &u2NemState,
4653 fRamExists ? &pRam->uNemRange : &pRamNew->uNemRange);
4654 if (u2NemState != UINT8_MAX)
4655 pgmPhysSetNemStateForPages(&pRamNew->aPages[idxFirstRamPage], cGuestPages, u2NemState);
4656 if (RT_SUCCESS(rc))
4657 return rc;
4658 }
4659 else
4660#endif
4661 return rc;
4662
4663 /*
4664 * bail out
4665 */
4666#ifdef VBOX_WITH_NATIVE_NEM
4667 /* unlink */
4668 if (pRomPrev)
4669 {
4670 pRomPrev->pNextR3 = pRom;
4671 pRomPrev->pNextR0 = pRom ? pRom->pSelfR0 : NIL_RTR0PTR;
4672 }
4673 else
4674 {
4675 pVM->pgm.s.pRomRangesR3 = pRom;
4676 pVM->pgm.s.pRomRangesR0 = pRom ? pRom->pSelfR0 : NIL_RTR0PTR;
4677 }
4678
4679 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4680 {
4681# ifdef VBOX_WITH_PGM_NEM_MODE
4682 if (pVM->pgm.s.fNemMode)
4683 pVM->pgm.s.cPrivatePages -= cGuestPages;
4684 else
4685# endif
4686 pVM->pgm.s.cZeroPages -= cGuestPages;
4687 pVM->pgm.s.cAllPages -= cGuestPages;
4688 }
4689#endif
4690 }
4691 else
4692 rc = VERR_NO_MEMORY;
4693 }
4694
4695 int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
4696 AssertRC(rc2);
4697 }
4698
4699 if (!fRamExists)
4700 pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
4701 else
4702 {
4703 PPGMPAGE pRamPage = &pRam->aPages[idxFirstRamPage];
4704#ifdef VBOX_WITH_PGM_NEM_MODE
4705 if (pVM->pgm.s.fNemMode)
4706 {
4707 Assert(pvRam == NULL); Assert(pReq == NULL);
4708 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4709 {
4710 Assert(PGM_PAGE_GET_HCPHYS(pRamPage) == UINT64_C(0x0000fffffffff000));
4711 Assert(PGM_PAGE_GET_PAGEID(pRamPage) == NIL_GMM_PAGEID);
4712 Assert(PGM_PAGE_GET_STATE(pRamPage) == PGM_PAGE_STATE_ALLOCATED);
4713 PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_RAM);
4714 PGM_PAGE_SET_STATE(pVM, pRamPage, PGM_PAGE_STATE_ALLOCATED);
4715 }
4716 }
4717 else
4718#endif
4719 {
4720 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++)
4721 PGM_PAGE_INIT_ZERO(pRamPage, pVM, PGMPAGETYPE_RAM);
4722 pVM->pgm.s.cZeroPages += cGuestPages;
4723 pVM->pgm.s.cPrivatePages -= cGuestPages;
4724 }
4725 }
4726
4727 SUPR3PageFreeEx(pRomNew, cRangePages);
4728 }
4729
4730 /** @todo Purge the mapping cache or something... */
4731#ifdef VBOX_WITH_PGM_NEM_MODE
4732 if (pVM->pgm.s.fNemMode)
4733 {
4734 Assert(!pReq);
4735 if (pvRam)
4736 SUPR3PageFree(pvRam, cHostPages);
4737 if (pvAlt)
4738 SUPR3PageFree(pvAlt, cHostPages);
4739 }
4740 else
4741#endif
4742 {
4743 GMMR3FreeAllocatedPages(pVM, pReq);
4744 GMMR3AllocatePagesCleanup(pReq);
4745 }
4746 return rc;
4747}
4748
4749
4750/**
4751 * Registers a ROM image.
4752 *
4753 * Shadowed ROM images requires double the amount of backing memory, so,
4754 * don't use that unless you have to. Shadowing of ROM images is process
4755 * where we can select where the reads go and where the writes go. On real
4756 * hardware the chipset provides means to configure this. We provide
4757 * PGMR3PhysProtectROM() for this purpose.
4758 *
4759 * A read-only copy of the ROM image will always be kept around while we
4760 * will allocate RAM pages for the changes on demand (unless all memory
4761 * is configured to be preallocated).
4762 *
4763 * @returns VBox status code.
4764 * @param pVM The cross context VM structure.
4765 * @param pDevIns The device instance owning the ROM.
4766 * @param GCPhys First physical address in the range.
4767 * Must be page aligned!
4768 * @param cb The size of the range (in bytes).
4769 * Must be page aligned!
4770 * @param pvBinary Pointer to the binary data backing the ROM image.
4771 * @param cbBinary The size of the binary data pvBinary points to.
4772 * This must be less or equal to @a cb.
4773 * @param fFlags Mask of flags, PGMPHYS_ROM_FLAGS_XXX.
4774 * @param pszDesc Pointer to description string. This must not be freed.
4775 *
4776 * @remark There is no way to remove the rom, automatically on device cleanup or
4777 * manually from the device yet. This isn't difficult in any way, it's
4778 * just not something we expect to be necessary for a while.
4779 */
4780VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
4781 const void *pvBinary, uint32_t cbBinary, uint8_t fFlags, const char *pszDesc)
4782{
4783 Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
4784 pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
4785 PGM_LOCK_VOID(pVM);
4786 int rc = pgmR3PhysRomRegisterLocked(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
4787 PGM_UNLOCK(pVM);
4788 return rc;
4789}
4790
4791
4792/**
4793 * Called by PGMR3MemSetup to reset the shadow, switch to the virgin, and verify
4794 * that the virgin part is untouched.
4795 *
4796 * This is done after the normal memory has been cleared.
4797 *
4798 * ASSUMES that the caller owns the PGM lock.
4799 *
4800 * @param pVM The cross context VM structure.
4801 */
4802int pgmR3PhysRomReset(PVM pVM)
4803{
4804 PGM_LOCK_ASSERT_OWNER(pVM);
4805 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
4806 {
4807 const uint32_t cGuestPages = pRom->cb >> GUEST_PAGE_SHIFT;
4808
4809 if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4810 {
4811 /*
4812 * Reset the physical handler.
4813 */
4814 int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
4815 AssertRCReturn(rc, rc);
4816
4817 /*
4818 * What we do with the shadow pages depends on the memory
4819 * preallocation option. If not enabled, we'll just throw
4820 * out all the dirty pages and replace them by the zero page.
4821 */
4822#ifdef VBOX_WITH_PGM_NEM_MODE
4823 if (pVM->pgm.s.fNemMode)
4824 {
4825 /* Clear all the shadow pages (currently using alternate backing). */
4826 RT_BZERO(pRom->pbR3Alternate, pRom->cb);
4827 }
4828 else
4829#endif
4830 if (!pVM->pgm.s.fRamPreAlloc)
4831 {
4832 /* Free the dirty pages. */
4833 uint32_t cPendingPages = 0;
4834 PGMMFREEPAGESREQ pReq;
4835 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4836 AssertRCReturn(rc, rc);
4837
4838 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++)
4839 if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
4840 && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
4841 {
4842 Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
4843 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
4844 pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT),
4845 (PGMPAGETYPE)PGM_PAGE_GET_TYPE(&pRom->aPages[iPage].Shadow));
4846 AssertLogRelRCReturn(rc, rc);
4847 }
4848
4849 if (cPendingPages)
4850 {
4851 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
4852 AssertLogRelRCReturn(rc, rc);
4853 }
4854 GMMR3FreePagesCleanup(pReq);
4855 }
4856 else
4857 {
4858 /* clear all the shadow pages. */
4859 for (uint32_t iPage = 0; iPage < cGuestPages; iPage++)
4860 {
4861 if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
4862 continue;
4863 Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
4864 void *pvDstPage;
4865 const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT);
4866 rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
4867 if (RT_FAILURE(rc))
4868 break;
4869 RT_BZERO(pvDstPage, GUEST_PAGE_SIZE);
4870 }
4871 AssertRCReturn(rc, rc);
4872 }
4873 }
4874
4875 /*
4876 * Restore the original ROM pages after a saved state load.
4877 * Also, in strict builds check that ROM pages remain unmodified.
4878 */
4879#ifndef VBOX_STRICT
4880 if (pVM->pgm.s.fRestoreRomPagesOnReset)
4881#endif
4882 {
4883 size_t cbSrcLeft = pRom->cbOriginal;
4884 uint8_t const *pbSrcPage = (uint8_t const *)pRom->pvOriginal;
4885 uint32_t cRestored = 0;
4886 for (uint32_t iPage = 0; iPage < cGuestPages && cbSrcLeft > 0; iPage++, pbSrcPage += GUEST_PAGE_SIZE)
4887 {
4888 const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT);
4889 PPGMPAGE const pPage = pgmPhysGetPage(pVM, GCPhys);
4890 void const *pvDstPage = NULL;
4891 int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvDstPage);
4892 if (RT_FAILURE(rc))
4893 break;
4894
4895 if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, GUEST_PAGE_SIZE)))
4896 {
4897 if (pVM->pgm.s.fRestoreRomPagesOnReset)
4898 {
4899 void *pvDstPageW = NULL;
4900 rc = pgmPhysPageMap(pVM, pPage, GCPhys, &pvDstPageW);
4901 AssertLogRelRCReturn(rc, rc);
4902 memcpy(pvDstPageW, pbSrcPage, RT_MIN(cbSrcLeft, GUEST_PAGE_SIZE));
4903 cRestored++;
4904 }
4905 else
4906 LogRel(("pgmR3PhysRomReset: %RGp: ROM page changed (%s)\n", GCPhys, pRom->pszDesc));
4907 }
4908 cbSrcLeft -= RT_MIN(cbSrcLeft, GUEST_PAGE_SIZE);
4909 }
4910 if (cRestored > 0)
4911 LogRel(("PGM: ROM \"%s\": Reloaded %u of %u pages.\n", pRom->pszDesc, cRestored, cGuestPages));
4912 }
4913 }
4914
4915 /* Clear the ROM restore flag now as we only need to do this once after
4916 loading saved state. */
4917 pVM->pgm.s.fRestoreRomPagesOnReset = false;
4918
4919 return VINF_SUCCESS;
4920}
4921
4922
4923/**
4924 * Called by PGMR3Term to free resources.
4925 *
4926 * ASSUMES that the caller owns the PGM lock.
4927 *
4928 * @param pVM The cross context VM structure.
4929 */
4930void pgmR3PhysRomTerm(PVM pVM)
4931{
4932 /*
4933 * Free the heap copy of the original bits.
4934 */
4935 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
4936 {
4937 if ( pRom->pvOriginal
4938 && !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
4939 {
4940 RTMemFree((void *)pRom->pvOriginal);
4941 pRom->pvOriginal = NULL;
4942 }
4943 }
4944}
4945
4946
4947/**
4948 * Change the shadowing of a range of ROM pages.
4949 *
4950 * This is intended for implementing chipset specific memory registers
4951 * and will not be very strict about the input. It will silently ignore
4952 * any pages that are not the part of a shadowed ROM.
4953 *
4954 * @returns VBox status code.
4955 * @retval VINF_PGM_SYNC_CR3
4956 *
4957 * @param pVM The cross context VM structure.
4958 * @param GCPhys Where to start. Page aligned.
4959 * @param cb How much to change. Page aligned.
4960 * @param enmProt The new ROM protection.
4961 */
4962VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
4963{
4964 /*
4965 * Check input
4966 */
4967 if (!cb)
4968 return VINF_SUCCESS;
4969 AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
4970 AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
4971 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
4972 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
4973 AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
4974
4975 /*
4976 * Process the request.
4977 */
4978 PGM_LOCK_VOID(pVM);
4979 int rc = VINF_SUCCESS;
4980 bool fFlushTLB = false;
4981 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
4982 {
4983 if ( GCPhys <= pRom->GCPhysLast
4984 && GCPhysLast >= pRom->GCPhys
4985 && (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
4986 {
4987 /*
4988 * Iterate the relevant pages and make necessary the changes.
4989 */
4990#ifdef VBOX_WITH_NATIVE_NEM
4991 PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhys);
4992 AssertPtrReturn(pRam, VERR_INTERNAL_ERROR_3);
4993#endif
4994 bool fChanges = false;
4995 uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
4996 ? pRom->cb >> GUEST_PAGE_SHIFT
4997 : (GCPhysLast - pRom->GCPhys + 1) >> GUEST_PAGE_SHIFT;
4998 for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
4999 iPage < cPages;
5000 iPage++)
5001 {
5002 PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
5003 if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
5004 {
5005 fChanges = true;
5006
5007 /* flush references to the page. */
5008 RTGCPHYS const GCPhysPage = pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT);
5009 PPGMPAGE pRamPage = pgmPhysGetPage(pVM, GCPhysPage);
5010 int rc2 = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pRamPage, true /*fFlushPTEs*/, &fFlushTLB);
5011 if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS || RT_FAILURE(rc2)))
5012 rc = rc2;
5013#ifdef VBOX_WITH_NATIVE_NEM
5014 uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pRamPage);
5015#endif
5016
5017 PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
5018 PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
5019
5020 *pOld = *pRamPage;
5021 *pRamPage = *pNew;
5022 /** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
5023
5024#ifdef VBOX_WITH_NATIVE_NEM
5025# ifdef VBOX_WITH_PGM_NEM_MODE
5026 /* In simplified mode we have to switch the page data around too. */
5027 if (pVM->pgm.s.fNemMode)
5028 {
5029 uint8_t abPage[GUEST_PAGE_SIZE];
5030 uint8_t * const pbRamPage = PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage);
5031 memcpy(abPage, &pRom->pbR3Alternate[(size_t)iPage << GUEST_PAGE_SHIFT], sizeof(abPage));
5032 memcpy(&pRom->pbR3Alternate[(size_t)iPage << GUEST_PAGE_SHIFT], pbRamPage, sizeof(abPage));
5033 memcpy(pbRamPage, abPage, sizeof(abPage));
5034 }
5035# endif
5036 /* Tell NEM about the backing and protection change. */
5037 if (VM_IS_NEM_ENABLED(pVM))
5038 {
5039 PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pNew);
5040 NEMHCNotifyPhysPageChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pOld), PGM_PAGE_GET_HCPHYS(pNew),
5041 PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage),
5042 pgmPhysPageCalcNemProtection(pRamPage, enmType), enmType, &u2State);
5043 PGM_PAGE_SET_NEM_STATE(pRamPage, u2State);
5044 }
5045#endif
5046 }
5047 pRomPage->enmProt = enmProt;
5048 }
5049
5050 /*
5051 * Reset the access handler if we made changes, no need
5052 * to optimize this.
5053 */
5054 if (fChanges)
5055 {
5056 int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
5057 if (RT_FAILURE(rc2))
5058 {
5059 PGM_UNLOCK(pVM);
5060 AssertRC(rc);
5061 return rc2;
5062 }
5063 }
5064
5065 /* Advance - cb isn't updated. */
5066 GCPhys = pRom->GCPhys + (cPages << GUEST_PAGE_SHIFT);
5067 }
5068 }
5069 PGM_UNLOCK(pVM);
5070 if (fFlushTLB)
5071 PGM_INVL_ALL_VCPU_TLBS(pVM);
5072
5073 return rc;
5074}
5075
5076
5077
5078/*********************************************************************************************************************************
5079* Ballooning *
5080*********************************************************************************************************************************/
5081
5082#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
5083
5084/**
5085 * Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
5086 *
5087 * This is only called on one of the EMTs while the other ones are waiting for
5088 * it to complete this function.
5089 *
5090 * @returns VINF_SUCCESS (VBox strict status code).
5091 * @param pVM The cross context VM structure.
5092 * @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
5093 * @param pvUser User parameter
5094 */
5095static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
5096{
5097 uintptr_t *paUser = (uintptr_t *)pvUser;
5098 bool fInflate = !!paUser[0];
5099 unsigned cPages = paUser[1];
5100 RTGCPHYS *paPhysPage = (RTGCPHYS *)paUser[2];
5101 uint32_t cPendingPages = 0;
5102 PGMMFREEPAGESREQ pReq;
5103 int rc;
5104
5105 Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
5106 PGM_LOCK_VOID(pVM);
5107
5108 if (fInflate)
5109 {
5110 /* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
5111 pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
5112
5113 /* Replace pages with ZERO pages. */
5114 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
5115 if (RT_FAILURE(rc))
5116 {
5117 PGM_UNLOCK(pVM);
5118 AssertLogRelRC(rc);
5119 return rc;
5120 }
5121
5122 /* Iterate the pages. */
5123 for (unsigned i = 0; i < cPages; i++)
5124 {
5125 PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
5126 if ( pPage == NULL
5127 || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
5128 {
5129 Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
5130 break;
5131 }
5132
5133 LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
5134
5135 /* Flush the shadow PT if this page was previously used as a guest page table. */
5136 pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
5137
5138 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i], (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
5139 if (RT_FAILURE(rc))
5140 {
5141 PGM_UNLOCK(pVM);
5142 AssertLogRelRC(rc);
5143 return rc;
5144 }
5145 Assert(PGM_PAGE_IS_ZERO(pPage));
5146 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
5147 }
5148
5149 if (cPendingPages)
5150 {
5151 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
5152 if (RT_FAILURE(rc))
5153 {
5154 PGM_UNLOCK(pVM);
5155 AssertLogRelRC(rc);
5156 return rc;
5157 }
5158 }
5159 GMMR3FreePagesCleanup(pReq);
5160 }
5161 else
5162 {
5163 /* Iterate the pages. */
5164 for (unsigned i = 0; i < cPages; i++)
5165 {
5166 PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
5167 AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
5168
5169 LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
5170
5171 Assert(PGM_PAGE_IS_BALLOONED(pPage));
5172
5173 /* Change back to zero page. (NEM does not need to be informed.) */
5174 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
5175 }
5176
5177 /* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
5178 }
5179
5180 /* Notify GMM about the balloon change. */
5181 rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
5182 if (RT_SUCCESS(rc))
5183 {
5184 if (!fInflate)
5185 {
5186 Assert(pVM->pgm.s.cBalloonedPages >= cPages);
5187 pVM->pgm.s.cBalloonedPages -= cPages;
5188 }
5189 else
5190 pVM->pgm.s.cBalloonedPages += cPages;
5191 }
5192
5193 PGM_UNLOCK(pVM);
5194
5195 /* Flush the recompiler's TLB as well. */
5196 for (VMCPUID i = 0; i < pVM->cCpus; i++)
5197 CPUMSetChangedFlags(pVM->apCpusR3[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5198
5199 AssertLogRelRC(rc);
5200 return rc;
5201}
5202
5203
5204/**
5205 * Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
5206 *
5207 * @returns VBox status code.
5208 * @param pVM The cross context VM structure.
5209 * @param fInflate Inflate or deflate memory balloon
5210 * @param cPages Number of pages to free
5211 * @param paPhysPage Array of guest physical addresses
5212 */
5213static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
5214{
5215 uintptr_t paUser[3];
5216
5217 paUser[0] = fInflate;
5218 paUser[1] = cPages;
5219 paUser[2] = (uintptr_t)paPhysPage;
5220 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
5221 AssertRC(rc);
5222
5223 /* Made a copy in PGMR3PhysFreeRamPages; free it here. */
5224 RTMemFree(paPhysPage);
5225}
5226
5227#endif /* 64-bit host && (Windows || Solaris || Linux || FreeBSD) */
5228
5229/**
5230 * Inflate or deflate a memory balloon
5231 *
5232 * @returns VBox status code.
5233 * @param pVM The cross context VM structure.
5234 * @param fInflate Inflate or deflate memory balloon
5235 * @param cPages Number of pages to free
5236 * @param paPhysPage Array of guest physical addresses
5237 */
5238VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
5239{
5240 /* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
5241#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
5242 int rc;
5243
5244 /* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
5245 AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
5246
5247 /* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
5248 * In the SMP case we post a request packet to postpone the job.
5249 */
5250 if (pVM->cCpus > 1)
5251 {
5252 unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
5253 RTGCPHYS *paPhysPageCopy = (RTGCPHYS *)RTMemAlloc(cbPhysPage);
5254 AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
5255
5256 memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
5257
5258 rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
5259 AssertRC(rc);
5260 }
5261 else
5262 {
5263 uintptr_t paUser[3];
5264
5265 paUser[0] = fInflate;
5266 paUser[1] = cPages;
5267 paUser[2] = (uintptr_t)paPhysPage;
5268 rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
5269 AssertRC(rc);
5270 }
5271 return rc;
5272
5273#else
5274 NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
5275 return VERR_NOT_IMPLEMENTED;
5276#endif
5277}
5278
5279
5280/*********************************************************************************************************************************
5281* Write Monitoring *
5282*********************************************************************************************************************************/
5283
5284/**
5285 * Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
5286 * physical RAM.
5287 *
5288 * This is only called on one of the EMTs while the other ones are waiting for
5289 * it to complete this function.
5290 *
5291 * @returns VINF_SUCCESS (VBox strict status code).
5292 * @param pVM The cross context VM structure.
5293 * @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
5294 * @param pvUser User parameter, unused.
5295 */
5296static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
5297{
5298 int rc = VINF_SUCCESS;
5299 NOREF(pvUser); NOREF(pVCpu);
5300
5301 PGM_LOCK_VOID(pVM);
5302#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5303 pgmPoolResetDirtyPages(pVM);
5304#endif
5305
5306 /** @todo pointless to write protect the physical page pointed to by RSP. */
5307
5308 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
5309 pRam;
5310 pRam = pRam->CTX_SUFF(pNext))
5311 {
5312 uint32_t cPages = pRam->cb >> GUEST_PAGE_SHIFT;
5313 for (uint32_t iPage = 0; iPage < cPages; iPage++)
5314 {
5315 PPGMPAGE pPage = &pRam->aPages[iPage];
5316 PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
5317
5318 if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
5319 || enmPageType == PGMPAGETYPE_MMIO2)
5320 {
5321 /*
5322 * A RAM page.
5323 */
5324 switch (PGM_PAGE_GET_STATE(pPage))
5325 {
5326 case PGM_PAGE_STATE_ALLOCATED:
5327 /** @todo Optimize this: Don't always re-enable write
5328 * monitoring if the page is known to be very busy. */
5329 if (PGM_PAGE_IS_WRITTEN_TO(pPage))
5330 PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
5331
5332 pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT));
5333 break;
5334
5335 case PGM_PAGE_STATE_SHARED:
5336 AssertFailed();
5337 break;
5338
5339 case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
5340 default:
5341 break;
5342 }
5343 }
5344 }
5345 }
5346 pgmR3PoolWriteProtectPages(pVM);
5347 PGM_INVL_ALL_VCPU_TLBS(pVM);
5348 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
5349 CPUMSetChangedFlags(pVM->apCpusR3[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5350
5351 PGM_UNLOCK(pVM);
5352 return rc;
5353}
5354
5355/**
5356 * Protect all physical RAM to monitor writes
5357 *
5358 * @returns VBox status code.
5359 * @param pVM The cross context VM structure.
5360 */
5361VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
5362{
5363 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
5364
5365 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
5366 AssertRC(rc);
5367 return rc;
5368}
5369
5370
5371/*********************************************************************************************************************************
5372* Stats. *
5373*********************************************************************************************************************************/
5374
5375/**
5376 * Query the amount of free memory inside VMMR0
5377 *
5378 * @returns VBox status code.
5379 * @param pUVM The user mode VM handle.
5380 * @param pcbAllocMem Where to return the amount of memory allocated
5381 * by VMs.
5382 * @param pcbFreeMem Where to return the amount of memory that is
5383 * allocated from the host but not currently used
5384 * by any VMs.
5385 * @param pcbBallonedMem Where to return the sum of memory that is
5386 * currently ballooned by the VMs.
5387 * @param pcbSharedMem Where to return the amount of memory that is
5388 * currently shared.
5389 */
5390VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PUVM pUVM, uint64_t *pcbAllocMem, uint64_t *pcbFreeMem,
5391 uint64_t *pcbBallonedMem, uint64_t *pcbSharedMem)
5392{
5393 UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
5394 VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
5395
5396 uint64_t cAllocPages = 0;
5397 uint64_t cFreePages = 0;
5398 uint64_t cBalloonPages = 0;
5399 uint64_t cSharedPages = 0;
5400 if (!SUPR3IsDriverless())
5401 {
5402 int rc = GMMR3QueryHypervisorMemoryStats(pUVM->pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
5403 AssertRCReturn(rc, rc);
5404 }
5405
5406 if (pcbAllocMem)
5407 *pcbAllocMem = cAllocPages * _4K;
5408
5409 if (pcbFreeMem)
5410 *pcbFreeMem = cFreePages * _4K;
5411
5412 if (pcbBallonedMem)
5413 *pcbBallonedMem = cBalloonPages * _4K;
5414
5415 if (pcbSharedMem)
5416 *pcbSharedMem = cSharedPages * _4K;
5417
5418 Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
5419 cAllocPages, cFreePages, cBalloonPages, cSharedPages));
5420 return VINF_SUCCESS;
5421}
5422
5423
5424/**
5425 * Query memory stats for the VM.
5426 *
5427 * @returns VBox status code.
5428 * @param pUVM The user mode VM handle.
5429 * @param pcbTotalMem Where to return total amount memory the VM may
5430 * possibly use.
5431 * @param pcbPrivateMem Where to return the amount of private memory
5432 * currently allocated.
5433 * @param pcbSharedMem Where to return the amount of actually shared
5434 * memory currently used by the VM.
5435 * @param pcbZeroMem Where to return the amount of memory backed by
5436 * zero pages.
5437 *
5438 * @remarks The total mem is normally larger than the sum of the three
5439 * components. There are two reasons for this, first the amount of
5440 * shared memory is what we're sure is shared instead of what could
5441 * possibly be shared with someone. Secondly, because the total may
5442 * include some pure MMIO pages that doesn't go into any of the three
5443 * sub-counts.
5444 *
5445 * @todo Why do we return reused shared pages instead of anything that could
5446 * potentially be shared? Doesn't this mean the first VM gets a much
5447 * lower number of shared pages?
5448 */
5449VMMR3DECL(int) PGMR3QueryMemoryStats(PUVM pUVM, uint64_t *pcbTotalMem, uint64_t *pcbPrivateMem,
5450 uint64_t *pcbSharedMem, uint64_t *pcbZeroMem)
5451{
5452 UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
5453 PVM pVM = pUVM->pVM;
5454 VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
5455
5456 if (pcbTotalMem)
5457 *pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages * GUEST_PAGE_SIZE;
5458
5459 if (pcbPrivateMem)
5460 *pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages * GUEST_PAGE_SIZE;
5461
5462 if (pcbSharedMem)
5463 *pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages * GUEST_PAGE_SIZE;
5464
5465 if (pcbZeroMem)
5466 *pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages * GUEST_PAGE_SIZE;
5467
5468 Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
5469 return VINF_SUCCESS;
5470}
5471
5472
5473
5474/*********************************************************************************************************************************
5475* Chunk Mappings and Page Allocation *
5476*********************************************************************************************************************************/
5477
5478/**
5479 * Tree enumeration callback for dealing with age rollover.
5480 * It will perform a simple compression of the current age.
5481 */
5482static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
5483{
5484 /* Age compression - ASSUMES iNow == 4. */
5485 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
5486 if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
5487 pChunk->iLastUsed = 3;
5488 else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
5489 pChunk->iLastUsed = 2;
5490 else if (pChunk->iLastUsed)
5491 pChunk->iLastUsed = 1;
5492 else /* iLastUsed = 0 */
5493 pChunk->iLastUsed = 4;
5494
5495 NOREF(pvUser);
5496 return 0;
5497}
5498
5499
5500/**
5501 * The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
5502 */
5503typedef struct PGMR3PHYSCHUNKUNMAPCB
5504{
5505 PVM pVM; /**< Pointer to the VM. */
5506 PPGMCHUNKR3MAP pChunk; /**< The chunk to unmap. */
5507} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
5508
5509
5510/**
5511 * Callback used to find the mapping that's been unused for
5512 * the longest time.
5513 */
5514static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
5515{
5516 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
5517 PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
5518
5519 /*
5520 * Check for locks and compare when last used.
5521 */
5522 if (pChunk->cRefs)
5523 return 0;
5524 if (pChunk->cPermRefs)
5525 return 0;
5526 if ( pArg->pChunk
5527 && pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
5528 return 0;
5529
5530 /*
5531 * Check that it's not in any of the TLBs.
5532 */
5533 PVM pVM = pArg->pVM;
5534 if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
5535 == pChunk->Core.Key)
5536 {
5537 pChunk = NULL;
5538 return 0;
5539 }
5540#ifdef VBOX_STRICT
5541 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
5542 {
5543 Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
5544 Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
5545 }
5546#endif
5547
5548 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
5549 if (pVM->pgm.s.PhysTlbR3.aEntries[i].pMap == pChunk)
5550 return 0;
5551
5552 pArg->pChunk = pChunk;
5553 return 0;
5554}
5555
5556
5557/**
5558 * Finds a good candidate for unmapping when the ring-3 mapping cache is full.
5559 *
5560 * The candidate will not be part of any TLBs, so no need to flush
5561 * anything afterwards.
5562 *
5563 * @returns Chunk id.
5564 * @param pVM The cross context VM structure.
5565 */
5566static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
5567{
5568 PGM_LOCK_ASSERT_OWNER(pVM);
5569
5570 /*
5571 * Enumerate the age tree starting with the left most node.
5572 */
5573 STAM_PROFILE_START(&pVM->pgm.s.Stats.StatChunkFindCandidate, a);
5574 PGMR3PHYSCHUNKUNMAPCB Args;
5575 Args.pVM = pVM;
5576 Args.pChunk = NULL;
5577 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
5578 Assert(Args.pChunk);
5579 if (Args.pChunk)
5580 {
5581 Assert(Args.pChunk->cRefs == 0);
5582 Assert(Args.pChunk->cPermRefs == 0);
5583 STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkFindCandidate, a);
5584 return Args.pChunk->Core.Key;
5585 }
5586
5587 STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkFindCandidate, a);
5588 return INT32_MAX;
5589}
5590
5591
5592/**
5593 * Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
5594 *
5595 * This is only called on one of the EMTs while the other ones are waiting for
5596 * it to complete this function.
5597 *
5598 * @returns VINF_SUCCESS (VBox strict status code).
5599 * @param pVM The cross context VM structure.
5600 * @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
5601 * @param pvUser User pointer. Unused
5602 *
5603 */
5604static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
5605{
5606 int rc = VINF_SUCCESS;
5607 PGM_LOCK_VOID(pVM);
5608 NOREF(pVCpu); NOREF(pvUser);
5609
5610 if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
5611 {
5612 /* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
5613 /** @todo also not really efficient to unmap a chunk that contains PD
5614 * or PT pages. */
5615 pgmR3PoolClearAllRendezvous(pVM, pVM->apCpusR3[0], NULL /* no need to flush the REM TLB as we already did that above */);
5616
5617 /*
5618 * Request the ring-0 part to unmap a chunk to make space in the mapping cache.
5619 */
5620 GMMMAPUNMAPCHUNKREQ Req;
5621 Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
5622 Req.Hdr.cbReq = sizeof(Req);
5623 Req.pvR3 = NULL;
5624 Req.idChunkMap = NIL_GMM_CHUNKID;
5625 Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
5626 if (Req.idChunkUnmap != INT32_MAX)
5627 {
5628 STAM_PROFILE_START(&pVM->pgm.s.Stats.StatChunkUnmap, a);
5629 rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
5630 STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkUnmap, a);
5631 if (RT_SUCCESS(rc))
5632 {
5633 /*
5634 * Remove the unmapped one.
5635 */
5636 PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
5637 AssertRelease(pUnmappedChunk);
5638 AssertRelease(!pUnmappedChunk->cRefs);
5639 AssertRelease(!pUnmappedChunk->cPermRefs);
5640 pUnmappedChunk->pv = NULL;
5641 pUnmappedChunk->Core.Key = UINT32_MAX;
5642 MMR3HeapFree(pUnmappedChunk);
5643 pVM->pgm.s.ChunkR3Map.c--;
5644 pVM->pgm.s.cUnmappedChunks++;
5645
5646 /*
5647 * Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
5648 */
5649 /** @todo We should not flush chunks which include cr3 mappings. */
5650 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
5651 {
5652 PPGMCPU pPGM = &pVM->apCpusR3[idCpu]->pgm.s;
5653
5654 pPGM->pGst32BitPdR3 = NULL;
5655 pPGM->pGstPaePdptR3 = NULL;
5656 pPGM->pGstAmd64Pml4R3 = NULL;
5657 pPGM->pGstEptPml4R3 = NULL;
5658 pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
5659 pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
5660 pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
5661 pPGM->pGstEptPml4R0 = NIL_RTR0PTR;
5662 for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
5663 {
5664 pPGM->apGstPaePDsR3[i] = NULL;
5665 pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
5666 }
5667
5668 /* Flush REM TLBs. */
5669 CPUMSetChangedFlags(pVM->apCpusR3[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5670 }
5671 }
5672 }
5673 }
5674 PGM_UNLOCK(pVM);
5675 return rc;
5676}
5677
5678/**
5679 * Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
5680 *
5681 * @returns VBox status code.
5682 * @param pVM The cross context VM structure.
5683 */
5684static DECLCALLBACK(void) pgmR3PhysUnmapChunk(PVM pVM)
5685{
5686 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
5687 AssertRC(rc);
5688}
5689
5690
5691/**
5692 * Maps the given chunk into the ring-3 mapping cache.
5693 *
5694 * This will call ring-0.
5695 *
5696 * @returns VBox status code.
5697 * @param pVM The cross context VM structure.
5698 * @param idChunk The chunk in question.
5699 * @param ppChunk Where to store the chunk tracking structure.
5700 *
5701 * @remarks Called from within the PGM critical section.
5702 * @remarks Can be called from any thread!
5703 */
5704int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
5705{
5706 int rc;
5707
5708 PGM_LOCK_ASSERT_OWNER(pVM);
5709
5710 /*
5711 * Move the chunk time forward.
5712 */
5713 pVM->pgm.s.ChunkR3Map.iNow++;
5714 if (pVM->pgm.s.ChunkR3Map.iNow == 0)
5715 {
5716 pVM->pgm.s.ChunkR3Map.iNow = 4;
5717 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
5718 }
5719
5720 /*
5721 * Allocate a new tracking structure first.
5722 */
5723 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
5724 AssertReturn(pChunk, VERR_NO_MEMORY);
5725 pChunk->Core.Key = idChunk;
5726 pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
5727
5728 /*
5729 * Request the ring-0 part to map the chunk in question.
5730 */
5731 GMMMAPUNMAPCHUNKREQ Req;
5732 Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
5733 Req.Hdr.cbReq = sizeof(Req);
5734 Req.pvR3 = NULL;
5735 Req.idChunkMap = idChunk;
5736 Req.idChunkUnmap = NIL_GMM_CHUNKID;
5737
5738 /* Must be callable from any thread, so can't use VMMR3CallR0. */
5739 STAM_PROFILE_START(&pVM->pgm.s.Stats.StatChunkMap, a);
5740 rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
5741 STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkMap, a);
5742 if (RT_SUCCESS(rc))
5743 {
5744 pChunk->pv = Req.pvR3;
5745
5746 /*
5747 * If we're running out of virtual address space, then we should
5748 * unmap another chunk.
5749 *
5750 * Currently, an unmap operation requires that all other virtual CPUs
5751 * are idling and not by chance making use of the memory we're
5752 * unmapping. So, we create an async unmap operation here.
5753 *
5754 * Now, when creating or restoring a saved state this wont work very
5755 * well since we may want to restore all guest RAM + a little something.
5756 * So, we have to do the unmap synchronously. Fortunately for us
5757 * though, during these operations the other virtual CPUs are inactive
5758 * and it should be safe to do this.
5759 */
5760 /** @todo Eventually we should lock all memory when used and do
5761 * map+unmap as one kernel call without any rendezvous or
5762 * other precautions. */
5763 if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
5764 {
5765 switch (VMR3GetState(pVM))
5766 {
5767 case VMSTATE_LOADING:
5768 case VMSTATE_SAVING:
5769 {
5770 PVMCPU pVCpu = VMMGetCpu(pVM);
5771 if ( pVCpu
5772 && pVM->pgm.s.cDeprecatedPageLocks == 0)
5773 {
5774 pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
5775 break;
5776 }
5777 }
5778 RT_FALL_THRU();
5779 default:
5780 rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
5781 AssertRC(rc);
5782 break;
5783 }
5784 }
5785
5786 /*
5787 * Update the tree. We must do this after any unmapping to make sure
5788 * the chunk we're going to return isn't unmapped by accident.
5789 */
5790 AssertPtr(Req.pvR3);
5791 bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
5792 AssertRelease(fRc);
5793 pVM->pgm.s.ChunkR3Map.c++;
5794 pVM->pgm.s.cMappedChunks++;
5795 }
5796 else
5797 {
5798 /** @todo this may fail because of /proc/sys/vm/max_map_count, so we
5799 * should probably restrict ourselves on linux. */
5800 AssertRC(rc);
5801 MMR3HeapFree(pChunk);
5802 pChunk = NULL;
5803 }
5804
5805 *ppChunk = pChunk;
5806 return rc;
5807}
5808
5809
5810/**
5811 * Invalidates the TLB for the ring-3 mapping cache.
5812 *
5813 * @param pVM The cross context VM structure.
5814 */
5815VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
5816{
5817 PGM_LOCK_VOID(pVM);
5818 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
5819 {
5820 pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
5821 pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
5822 }
5823 /* The page map TLB references chunks, so invalidate that one too. */
5824 pgmPhysInvalidatePageMapTLB(pVM);
5825 PGM_UNLOCK(pVM);
5826}
5827
5828
5829/**
5830 * Response to VM_FF_PGM_NEED_HANDY_PAGES and helper for pgmPhysEnsureHandyPage.
5831 *
5832 * This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
5833 * signal and clear the out of memory condition. When called, this API is used
5834 * to try clear the condition when the user wants to resume.
5835 *
5836 * @returns The following VBox status codes.
5837 * @retval VINF_SUCCESS on success. FFs cleared.
5838 * @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
5839 * this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
5840 *
5841 * @param pVM The cross context VM structure.
5842 *
5843 * @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
5844 * in EM.cpp and shouldn't be propagated outside TRPM, HM, EM and
5845 * pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
5846 * handler.
5847 */
5848VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
5849{
5850 PGM_LOCK_VOID(pVM);
5851
5852 /*
5853 * Allocate more pages, noting down the index of the first new page.
5854 */
5855 uint32_t iClear = pVM->pgm.s.cHandyPages;
5856 AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
5857 Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
5858 int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
5859 /** @todo we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
5860 if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
5861 && pVM->pgm.s.cHandyPages > 0)
5862 {
5863 /* Still handy pages left, so don't panic. */
5864 rc = VINF_SUCCESS;
5865 }
5866
5867 if (RT_SUCCESS(rc))
5868 {
5869 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
5870 Assert(pVM->pgm.s.cHandyPages > 0);
5871#ifdef VBOX_STRICT
5872 uint32_t i;
5873 for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
5874 if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
5875 || pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
5876 || (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & GUEST_PAGE_OFFSET_MASK))
5877 break;
5878 if (i != pVM->pgm.s.cHandyPages)
5879 {
5880 RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
5881 RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
5882 for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
5883 RTAssertMsg2Add("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%s\n", j,
5884 pVM->pgm.s.aHandyPages[j].idPage,
5885 pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
5886 pVM->pgm.s.aHandyPages[j].idSharedPage,
5887 j == i ? " <---" : "");
5888 RTAssertPanic();
5889 }
5890#endif
5891 }
5892 else
5893 {
5894 /*
5895 * We should never get here unless there is a genuine shortage of
5896 * memory (or some internal error). Flag the error so the VM can be
5897 * suspended ASAP and the user informed. If we're totally out of
5898 * handy pages we will return failure.
5899 */
5900 /* Report the failure. */
5901 LogRel(("PGM: Failed to procure handy pages; rc=%Rrc cHandyPages=%#x\n"
5902 " cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
5903 rc, pVM->pgm.s.cHandyPages,
5904 pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cSharedPages, pVM->pgm.s.cZeroPages));
5905
5906 if ( rc != VERR_NO_MEMORY
5907 && rc != VERR_NO_PHYS_MEMORY
5908 && rc != VERR_LOCK_FAILED)
5909 for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
5910 {
5911 LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
5912 i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
5913 pVM->pgm.s.aHandyPages[i].idSharedPage));
5914 uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
5915 if (idPage != NIL_GMM_PAGEID)
5916 {
5917 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
5918 pRam;
5919 pRam = pRam->pNextR3)
5920 {
5921 uint32_t const cPages = pRam->cb >> GUEST_PAGE_SHIFT;
5922 for (uint32_t iPage = 0; iPage < cPages; iPage++)
5923 if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
5924 LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
5925 pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
5926 }
5927 }
5928 }
5929
5930 if (rc == VERR_NO_MEMORY)
5931 {
5932 uint64_t cbHostRamAvail = 0;
5933 int rc2 = RTSystemQueryAvailableRam(&cbHostRamAvail);
5934 if (RT_SUCCESS(rc2))
5935 LogRel(("Host RAM: %RU64MB available\n", cbHostRamAvail / _1M));
5936 else
5937 LogRel(("Cannot determine the amount of available host memory\n"));
5938 }
5939
5940 /* Set the FFs and adjust rc. */
5941 VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
5942 VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
5943 if ( rc == VERR_NO_MEMORY
5944 || rc == VERR_NO_PHYS_MEMORY
5945 || rc == VERR_LOCK_FAILED)
5946 rc = VINF_EM_NO_MEMORY;
5947 }
5948
5949 PGM_UNLOCK(pVM);
5950 return rc;
5951}
5952
5953
5954/*********************************************************************************************************************************
5955* Other Stuff *
5956*********************************************************************************************************************************/
5957
5958/**
5959 * Sets the Address Gate 20 state.
5960 *
5961 * @param pVCpu The cross context virtual CPU structure.
5962 * @param fEnable True if the gate should be enabled.
5963 * False if the gate should be disabled.
5964 */
5965VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
5966{
5967 LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
5968 if (pVCpu->pgm.s.fA20Enabled != fEnable)
5969 {
5970#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5971 PCCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
5972 if ( CPUMIsGuestInVmxRootMode(pCtx)
5973 && !fEnable)
5974 {
5975 Log(("Cannot enter A20M mode while in VMX root mode\n"));
5976 return;
5977 }
5978#endif
5979 pVCpu->pgm.s.fA20Enabled = fEnable;
5980 pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!fEnable << 20);
5981 if (VM_IS_NEM_ENABLED(pVCpu->CTX_SUFF(pVM)))
5982 NEMR3NotifySetA20(pVCpu, fEnable);
5983#ifdef PGM_WITH_A20
5984 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5985 pgmR3RefreshShadowModeAfterA20Change(pVCpu);
5986 HMFlushTlb(pVCpu);
5987#endif
5988#if 0 /* PGMGetPage will apply the A20 mask to the GCPhys it returns, so we must invalid both sides of the TLB. */
5989 IEMTlbInvalidateAllPhysical(pVCpu);
5990#else
5991 IEMTlbInvalidateAll(pVCpu);
5992#endif
5993 STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
5994 }
5995}
5996
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use