VirtualBox

About
Screenshots
Downloads
Documentation
End-user docs
Technical docs
Contribute
Community

Browse Source

Context Navigation

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 84044

Last change on this file since 84044 was 82968, checked in by vboxsync, 4 years ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 133.9 KB

Line
1	/* $Id: PGMAll.cpp 82968 2020-02-04 10:35:17Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/cpum.h>
25	#include <VBox/vmm/selm.h>
26	#include <VBox/vmm/iem.h>
27	#include <VBox/vmm/iom.h>
28	#include <VBox/sup.h>
29	#include <VBox/vmm/mm.h>
30	#include <VBox/vmm/stam.h>
31	#include <VBox/vmm/trpm.h>
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/hm.h>
34	#include <VBox/vmm/hm_vmx.h>
35	#include "PGMInternal.h"
36	#include <VBox/vmm/vmcc.h>
37	#include "PGMInline.h"
38	#include <iprt/assert.h>
39	#include <iprt/asm-amd64-x86.h>
40	#include <iprt/string.h>
41	#include <VBox/log.h>
42	#include <VBox/param.h>
43	#include <VBox/err.h>
44
45
46	/*********************************************************************************************************************************
47	* Internal Functions *
48	*********************************************************************************************************************************/
49	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
50	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
51	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
52	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
53
54
55	/*
56	* Shadow - 32-bit mode
57	*/
58	#define PGM_SHW_TYPE PGM_TYPE_32BIT
59	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
60	#include "PGMAllShw.h"
61
62	/* Guest - real mode */
63	#define PGM_GST_TYPE PGM_TYPE_REAL
64	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
65	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
66	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
67	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
68	#include "PGMGstDefs.h"
69	#include "PGMAllGst.h"
70	#include "PGMAllBth.h"
71	#undef BTH_PGMPOOLKIND_PT_FOR_PT
72	#undef BTH_PGMPOOLKIND_ROOT
73	#undef PGM_BTH_NAME
74	#undef PGM_GST_TYPE
75	#undef PGM_GST_NAME
76
77	/* Guest - protected mode */
78	#define PGM_GST_TYPE PGM_TYPE_PROT
79	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
80	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
81	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
82	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
83	#include "PGMGstDefs.h"
84	#include "PGMAllGst.h"
85	#include "PGMAllBth.h"
86	#undef BTH_PGMPOOLKIND_PT_FOR_PT
87	#undef BTH_PGMPOOLKIND_ROOT
88	#undef PGM_BTH_NAME
89	#undef PGM_GST_TYPE
90	#undef PGM_GST_NAME
91
92	/* Guest - 32-bit mode */
93	#define PGM_GST_TYPE PGM_TYPE_32BIT
94	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
95	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
96	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
97	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
98	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
99	#include "PGMGstDefs.h"
100	#include "PGMAllGst.h"
101	#include "PGMAllBth.h"
102	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
103	#undef BTH_PGMPOOLKIND_PT_FOR_PT
104	#undef BTH_PGMPOOLKIND_ROOT
105	#undef PGM_BTH_NAME
106	#undef PGM_GST_TYPE
107	#undef PGM_GST_NAME
108
109	#undef PGM_SHW_TYPE
110	#undef PGM_SHW_NAME
111
112
113	/*
114	* Shadow - PAE mode
115	*/
116	#define PGM_SHW_TYPE PGM_TYPE_PAE
117	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
118	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
119	#include "PGMAllShw.h"
120
121	/* Guest - real mode */
122	#define PGM_GST_TYPE PGM_TYPE_REAL
123	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
124	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
125	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
126	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
127	#include "PGMGstDefs.h"
128	#include "PGMAllBth.h"
129	#undef BTH_PGMPOOLKIND_PT_FOR_PT
130	#undef BTH_PGMPOOLKIND_ROOT
131	#undef PGM_BTH_NAME
132	#undef PGM_GST_TYPE
133	#undef PGM_GST_NAME
134
135	/* Guest - protected mode */
136	#define PGM_GST_TYPE PGM_TYPE_PROT
137	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
138	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
139	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
140	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
141	#include "PGMGstDefs.h"
142	#include "PGMAllBth.h"
143	#undef BTH_PGMPOOLKIND_PT_FOR_PT
144	#undef BTH_PGMPOOLKIND_ROOT
145	#undef PGM_BTH_NAME
146	#undef PGM_GST_TYPE
147	#undef PGM_GST_NAME
148
149	/* Guest - 32-bit mode */
150	#define PGM_GST_TYPE PGM_TYPE_32BIT
151	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
152	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
153	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
154	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
155	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
156	#include "PGMGstDefs.h"
157	#include "PGMAllBth.h"
158	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
159	#undef BTH_PGMPOOLKIND_PT_FOR_PT
160	#undef BTH_PGMPOOLKIND_ROOT
161	#undef PGM_BTH_NAME
162	#undef PGM_GST_TYPE
163	#undef PGM_GST_NAME
164
165
166	/* Guest - PAE mode */
167	#define PGM_GST_TYPE PGM_TYPE_PAE
168	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
169	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
170	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
171	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
172	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
173	#include "PGMGstDefs.h"
174	#include "PGMAllGst.h"
175	#include "PGMAllBth.h"
176	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
177	#undef BTH_PGMPOOLKIND_PT_FOR_PT
178	#undef BTH_PGMPOOLKIND_ROOT
179	#undef PGM_BTH_NAME
180	#undef PGM_GST_TYPE
181	#undef PGM_GST_NAME
182
183	#undef PGM_SHW_TYPE
184	#undef PGM_SHW_NAME
185
186
187	/*
188	* Shadow - AMD64 mode
189	*/
190	#define PGM_SHW_TYPE PGM_TYPE_AMD64
191	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
192	#include "PGMAllShw.h"
193
194	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
195	/** @todo retire this hack. */
196	#define PGM_GST_TYPE PGM_TYPE_PROT
197	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
198	#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
199	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
200	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
201	#include "PGMGstDefs.h"
202	#include "PGMAllBth.h"
203	#undef BTH_PGMPOOLKIND_PT_FOR_PT
204	#undef BTH_PGMPOOLKIND_ROOT
205	#undef PGM_BTH_NAME
206	#undef PGM_GST_TYPE
207	#undef PGM_GST_NAME
208
209	#ifdef VBOX_WITH_64_BITS_GUESTS
210	/* Guest - AMD64 mode */
211	# define PGM_GST_TYPE PGM_TYPE_AMD64
212	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
213	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
214	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
215	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
216	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
217	# include "PGMGstDefs.h"
218	# include "PGMAllGst.h"
219	# include "PGMAllBth.h"
220	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
221	# undef BTH_PGMPOOLKIND_PT_FOR_PT
222	# undef BTH_PGMPOOLKIND_ROOT
223	# undef PGM_BTH_NAME
224	# undef PGM_GST_TYPE
225	# undef PGM_GST_NAME
226	#endif /* VBOX_WITH_64_BITS_GUESTS */
227
228	#undef PGM_SHW_TYPE
229	#undef PGM_SHW_NAME
230
231
232	/*
233	* Shadow - 32-bit nested paging mode.
234	*/
235	#define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
236	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
237	#include "PGMAllShw.h"
238
239	/* Guest - real mode */
240	#define PGM_GST_TYPE PGM_TYPE_REAL
241	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
242	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
243	#include "PGMGstDefs.h"
244	#include "PGMAllBth.h"
245	#undef PGM_BTH_NAME
246	#undef PGM_GST_TYPE
247	#undef PGM_GST_NAME
248
249	/* Guest - protected mode */
250	#define PGM_GST_TYPE PGM_TYPE_PROT
251	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
252	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
253	#include "PGMGstDefs.h"
254	#include "PGMAllBth.h"
255	#undef PGM_BTH_NAME
256	#undef PGM_GST_TYPE
257	#undef PGM_GST_NAME
258
259	/* Guest - 32-bit mode */
260	#define PGM_GST_TYPE PGM_TYPE_32BIT
261	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
262	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
263	#include "PGMGstDefs.h"
264	#include "PGMAllBth.h"
265	#undef PGM_BTH_NAME
266	#undef PGM_GST_TYPE
267	#undef PGM_GST_NAME
268
269	/* Guest - PAE mode */
270	#define PGM_GST_TYPE PGM_TYPE_PAE
271	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
272	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
273	#include "PGMGstDefs.h"
274	#include "PGMAllBth.h"
275	#undef PGM_BTH_NAME
276	#undef PGM_GST_TYPE
277	#undef PGM_GST_NAME
278
279	#ifdef VBOX_WITH_64_BITS_GUESTS
280	/* Guest - AMD64 mode */
281	# define PGM_GST_TYPE PGM_TYPE_AMD64
282	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
283	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
284	# include "PGMGstDefs.h"
285	# include "PGMAllBth.h"
286	# undef PGM_BTH_NAME
287	# undef PGM_GST_TYPE
288	# undef PGM_GST_NAME
289	#endif /* VBOX_WITH_64_BITS_GUESTS */
290
291	#undef PGM_SHW_TYPE
292	#undef PGM_SHW_NAME
293
294
295	/*
296	* Shadow - PAE nested paging mode.
297	*/
298	#define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
299	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
300	#include "PGMAllShw.h"
301
302	/* Guest - real mode */
303	#define PGM_GST_TYPE PGM_TYPE_REAL
304	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
305	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
306	#include "PGMGstDefs.h"
307	#include "PGMAllBth.h"
308	#undef PGM_BTH_NAME
309	#undef PGM_GST_TYPE
310	#undef PGM_GST_NAME
311
312	/* Guest - protected mode */
313	#define PGM_GST_TYPE PGM_TYPE_PROT
314	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
315	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
316	#include "PGMGstDefs.h"
317	#include "PGMAllBth.h"
318	#undef PGM_BTH_NAME
319	#undef PGM_GST_TYPE
320	#undef PGM_GST_NAME
321
322	/* Guest - 32-bit mode */
323	#define PGM_GST_TYPE PGM_TYPE_32BIT
324	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
325	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
326	#include "PGMGstDefs.h"
327	#include "PGMAllBth.h"
328	#undef PGM_BTH_NAME
329	#undef PGM_GST_TYPE
330	#undef PGM_GST_NAME
331
332	/* Guest - PAE mode */
333	#define PGM_GST_TYPE PGM_TYPE_PAE
334	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
335	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
336	#include "PGMGstDefs.h"
337	#include "PGMAllBth.h"
338	#undef PGM_BTH_NAME
339	#undef PGM_GST_TYPE
340	#undef PGM_GST_NAME
341
342	#ifdef VBOX_WITH_64_BITS_GUESTS
343	/* Guest - AMD64 mode */
344	# define PGM_GST_TYPE PGM_TYPE_AMD64
345	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
346	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
347	# include "PGMGstDefs.h"
348	# include "PGMAllBth.h"
349	# undef PGM_BTH_NAME
350	# undef PGM_GST_TYPE
351	# undef PGM_GST_NAME
352	#endif /* VBOX_WITH_64_BITS_GUESTS */
353
354	#undef PGM_SHW_TYPE
355	#undef PGM_SHW_NAME
356
357
358	/*
359	* Shadow - AMD64 nested paging mode.
360	*/
361	#define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
362	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
363	#include "PGMAllShw.h"
364
365	/* Guest - real mode */
366	#define PGM_GST_TYPE PGM_TYPE_REAL
367	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
368	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
369	#include "PGMGstDefs.h"
370	#include "PGMAllBth.h"
371	#undef PGM_BTH_NAME
372	#undef PGM_GST_TYPE
373	#undef PGM_GST_NAME
374
375	/* Guest - protected mode */
376	#define PGM_GST_TYPE PGM_TYPE_PROT
377	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
378	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
379	#include "PGMGstDefs.h"
380	#include "PGMAllBth.h"
381	#undef PGM_BTH_NAME
382	#undef PGM_GST_TYPE
383	#undef PGM_GST_NAME
384
385	/* Guest - 32-bit mode */
386	#define PGM_GST_TYPE PGM_TYPE_32BIT
387	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
388	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
389	#include "PGMGstDefs.h"
390	#include "PGMAllBth.h"
391	#undef PGM_BTH_NAME
392	#undef PGM_GST_TYPE
393	#undef PGM_GST_NAME
394
395	/* Guest - PAE mode */
396	#define PGM_GST_TYPE PGM_TYPE_PAE
397	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
398	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
399	#include "PGMGstDefs.h"
400	#include "PGMAllBth.h"
401	#undef PGM_BTH_NAME
402	#undef PGM_GST_TYPE
403	#undef PGM_GST_NAME
404
405	#ifdef VBOX_WITH_64_BITS_GUESTS
406	/* Guest - AMD64 mode */
407	# define PGM_GST_TYPE PGM_TYPE_AMD64
408	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
409	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
410	# include "PGMGstDefs.h"
411	# include "PGMAllBth.h"
412	# undef PGM_BTH_NAME
413	# undef PGM_GST_TYPE
414	# undef PGM_GST_NAME
415	#endif /* VBOX_WITH_64_BITS_GUESTS */
416
417	#undef PGM_SHW_TYPE
418	#undef PGM_SHW_NAME
419
420
421	/*
422	* Shadow - EPT.
423	*/
424	#define PGM_SHW_TYPE PGM_TYPE_EPT
425	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
426	#include "PGMAllShw.h"
427
428	/* Guest - real mode */
429	#define PGM_GST_TYPE PGM_TYPE_REAL
430	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
431	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
432	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
433	#include "PGMGstDefs.h"
434	#include "PGMAllBth.h"
435	#undef BTH_PGMPOOLKIND_PT_FOR_PT
436	#undef PGM_BTH_NAME
437	#undef PGM_GST_TYPE
438	#undef PGM_GST_NAME
439
440	/* Guest - protected mode */
441	#define PGM_GST_TYPE PGM_TYPE_PROT
442	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
443	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
444	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
445	#include "PGMGstDefs.h"
446	#include "PGMAllBth.h"
447	#undef BTH_PGMPOOLKIND_PT_FOR_PT
448	#undef PGM_BTH_NAME
449	#undef PGM_GST_TYPE
450	#undef PGM_GST_NAME
451
452	/* Guest - 32-bit mode */
453	#define PGM_GST_TYPE PGM_TYPE_32BIT
454	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
455	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
456	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
457	#include "PGMGstDefs.h"
458	#include "PGMAllBth.h"
459	#undef BTH_PGMPOOLKIND_PT_FOR_PT
460	#undef PGM_BTH_NAME
461	#undef PGM_GST_TYPE
462	#undef PGM_GST_NAME
463
464	/* Guest - PAE mode */
465	#define PGM_GST_TYPE PGM_TYPE_PAE
466	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
467	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
468	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
469	#include "PGMGstDefs.h"
470	#include "PGMAllBth.h"
471	#undef BTH_PGMPOOLKIND_PT_FOR_PT
472	#undef PGM_BTH_NAME
473	#undef PGM_GST_TYPE
474	#undef PGM_GST_NAME
475
476	#ifdef VBOX_WITH_64_BITS_GUESTS
477	/* Guest - AMD64 mode */
478	# define PGM_GST_TYPE PGM_TYPE_AMD64
479	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
480	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
481	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
482	# include "PGMGstDefs.h"
483	# include "PGMAllBth.h"
484	# undef BTH_PGMPOOLKIND_PT_FOR_PT
485	# undef PGM_BTH_NAME
486	# undef PGM_GST_TYPE
487	# undef PGM_GST_NAME
488	#endif /* VBOX_WITH_64_BITS_GUESTS */
489
490	#undef PGM_SHW_TYPE
491	#undef PGM_SHW_NAME
492
493
494	/*
495	* Shadow - NEM / None.
496	*/
497	#define PGM_SHW_TYPE PGM_TYPE_NONE
498	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
499	#include "PGMAllShw.h"
500
501	/* Guest - real mode */
502	#define PGM_GST_TYPE PGM_TYPE_REAL
503	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
504	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
505	#include "PGMGstDefs.h"
506	#include "PGMAllBth.h"
507	#undef PGM_BTH_NAME
508	#undef PGM_GST_TYPE
509	#undef PGM_GST_NAME
510
511	/* Guest - protected mode */
512	#define PGM_GST_TYPE PGM_TYPE_PROT
513	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
514	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
515	#include "PGMGstDefs.h"
516	#include "PGMAllBth.h"
517	#undef PGM_BTH_NAME
518	#undef PGM_GST_TYPE
519	#undef PGM_GST_NAME
520
521	/* Guest - 32-bit mode */
522	#define PGM_GST_TYPE PGM_TYPE_32BIT
523	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
524	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
525	#include "PGMGstDefs.h"
526	#include "PGMAllBth.h"
527	#undef PGM_BTH_NAME
528	#undef PGM_GST_TYPE
529	#undef PGM_GST_NAME
530
531	/* Guest - PAE mode */
532	#define PGM_GST_TYPE PGM_TYPE_PAE
533	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
534	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
535	#include "PGMGstDefs.h"
536	#include "PGMAllBth.h"
537	#undef PGM_BTH_NAME
538	#undef PGM_GST_TYPE
539	#undef PGM_GST_NAME
540
541	#ifdef VBOX_WITH_64_BITS_GUESTS
542	/* Guest - AMD64 mode */
543	# define PGM_GST_TYPE PGM_TYPE_AMD64
544	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
545	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
546	# include "PGMGstDefs.h"
547	# include "PGMAllBth.h"
548	# undef PGM_BTH_NAME
549	# undef PGM_GST_TYPE
550	# undef PGM_GST_NAME
551	#endif /* VBOX_WITH_64_BITS_GUESTS */
552
553	#undef PGM_SHW_TYPE
554	#undef PGM_SHW_NAME
555
556
557
558	/**
559	* Guest mode data array.
560	*/
561	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
562	{
563	{ UINT32_MAX, NULL, NULL, NULL, NULL, NULL }, /* 0 */
564	{
565	PGM_TYPE_REAL,
566	PGM_GST_NAME_REAL(GetPage),
567	PGM_GST_NAME_REAL(ModifyPage),
568	PGM_GST_NAME_REAL(GetPDE),
569	PGM_GST_NAME_REAL(Enter),
570	PGM_GST_NAME_REAL(Exit),
571	#ifdef IN_RING3
572	PGM_GST_NAME_REAL(Relocate),
573	#endif
574	},
575	{
576	PGM_TYPE_PROT,
577	PGM_GST_NAME_PROT(GetPage),
578	PGM_GST_NAME_PROT(ModifyPage),
579	PGM_GST_NAME_PROT(GetPDE),
580	PGM_GST_NAME_PROT(Enter),
581	PGM_GST_NAME_PROT(Exit),
582	#ifdef IN_RING3
583	PGM_GST_NAME_PROT(Relocate),
584	#endif
585	},
586	{
587	PGM_TYPE_32BIT,
588	PGM_GST_NAME_32BIT(GetPage),
589	PGM_GST_NAME_32BIT(ModifyPage),
590	PGM_GST_NAME_32BIT(GetPDE),
591	PGM_GST_NAME_32BIT(Enter),
592	PGM_GST_NAME_32BIT(Exit),
593	#ifdef IN_RING3
594	PGM_GST_NAME_32BIT(Relocate),
595	#endif
596	},
597	{
598	PGM_TYPE_PAE,
599	PGM_GST_NAME_PAE(GetPage),
600	PGM_GST_NAME_PAE(ModifyPage),
601	PGM_GST_NAME_PAE(GetPDE),
602	PGM_GST_NAME_PAE(Enter),
603	PGM_GST_NAME_PAE(Exit),
604	#ifdef IN_RING3
605	PGM_GST_NAME_PAE(Relocate),
606	#endif
607	},
608	#ifdef VBOX_WITH_64_BITS_GUESTS
609	{
610	PGM_TYPE_AMD64,
611	PGM_GST_NAME_AMD64(GetPage),
612	PGM_GST_NAME_AMD64(ModifyPage),
613	PGM_GST_NAME_AMD64(GetPDE),
614	PGM_GST_NAME_AMD64(Enter),
615	PGM_GST_NAME_AMD64(Exit),
616	# ifdef IN_RING3
617	PGM_GST_NAME_AMD64(Relocate),
618	# endif
619	},
620	#endif
621	};
622
623
624	/**
625	* The shadow mode data array.
626	*/
627	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
628	{
629	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
630	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
631	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
632	{
633	PGM_TYPE_32BIT,
634	PGM_SHW_NAME_32BIT(GetPage),
635	PGM_SHW_NAME_32BIT(ModifyPage),
636	PGM_SHW_NAME_32BIT(Enter),
637	PGM_SHW_NAME_32BIT(Exit),
638	#ifdef IN_RING3
639	PGM_SHW_NAME_32BIT(Relocate),
640	#endif
641	},
642	{
643	PGM_TYPE_PAE,
644	PGM_SHW_NAME_PAE(GetPage),
645	PGM_SHW_NAME_PAE(ModifyPage),
646	PGM_SHW_NAME_PAE(Enter),
647	PGM_SHW_NAME_PAE(Exit),
648	#ifdef IN_RING3
649	PGM_SHW_NAME_PAE(Relocate),
650	#endif
651	},
652	{
653	PGM_TYPE_AMD64,
654	PGM_SHW_NAME_AMD64(GetPage),
655	PGM_SHW_NAME_AMD64(ModifyPage),
656	PGM_SHW_NAME_AMD64(Enter),
657	PGM_SHW_NAME_AMD64(Exit),
658	#ifdef IN_RING3
659	PGM_SHW_NAME_AMD64(Relocate),
660	#endif
661	},
662	{
663	PGM_TYPE_NESTED_32BIT,
664	PGM_SHW_NAME_NESTED_32BIT(GetPage),
665	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
666	PGM_SHW_NAME_NESTED_32BIT(Enter),
667	PGM_SHW_NAME_NESTED_32BIT(Exit),
668	#ifdef IN_RING3
669	PGM_SHW_NAME_NESTED_32BIT(Relocate),
670	#endif
671	},
672	{
673	PGM_TYPE_NESTED_PAE,
674	PGM_SHW_NAME_NESTED_PAE(GetPage),
675	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
676	PGM_SHW_NAME_NESTED_PAE(Enter),
677	PGM_SHW_NAME_NESTED_PAE(Exit),
678	#ifdef IN_RING3
679	PGM_SHW_NAME_NESTED_PAE(Relocate),
680	#endif
681	},
682	{
683	PGM_TYPE_NESTED_AMD64,
684	PGM_SHW_NAME_NESTED_AMD64(GetPage),
685	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
686	PGM_SHW_NAME_NESTED_AMD64(Enter),
687	PGM_SHW_NAME_NESTED_AMD64(Exit),
688	#ifdef IN_RING3
689	PGM_SHW_NAME_NESTED_AMD64(Relocate),
690	#endif
691	},
692	{
693	PGM_TYPE_EPT,
694	PGM_SHW_NAME_EPT(GetPage),
695	PGM_SHW_NAME_EPT(ModifyPage),
696	PGM_SHW_NAME_EPT(Enter),
697	PGM_SHW_NAME_EPT(Exit),
698	#ifdef IN_RING3
699	PGM_SHW_NAME_EPT(Relocate),
700	#endif
701	},
702	{
703	PGM_TYPE_NONE,
704	PGM_SHW_NAME_NONE(GetPage),
705	PGM_SHW_NAME_NONE(ModifyPage),
706	PGM_SHW_NAME_NONE(Enter),
707	PGM_SHW_NAME_NONE(Exit),
708	#ifdef IN_RING3
709	PGM_SHW_NAME_NONE(Relocate),
710	#endif
711	},
712	};
713
714
715	/**
716	* The guest+shadow mode data array.
717	*/
718	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
719	{
720	#if !defined(IN_RING3) && !defined(VBOX_STRICT)
721	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
722	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
723	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler) }
724
725	#elif !defined(IN_RING3) && defined(VBOX_STRICT)
726	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
727	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
728	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(AssertCR3) }
729
730	#elif defined(IN_RING3) && !defined(VBOX_STRICT)
731	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
732	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
733	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
734
735	#elif defined(IN_RING3) && defined(VBOX_STRICT)
736	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
737	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
738	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
739
740	#else
741	# error "Misconfig."
742	#endif
743
744	/* 32-bit shadow paging mode: */
745	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
746	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
747	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
748	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
749	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
750	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
751	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
752	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
753	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
754	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
755	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
756
757	/* PAE shadow paging mode: */
758	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
759	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
760	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
761	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
762	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
763	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
764	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
765	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
766	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
767	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
768	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
769
770	/* AMD64 shadow paging mode: */
771	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
772	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
773	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
774	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
775	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
776	#ifdef VBOX_WITH_64_BITS_GUESTS
777	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
778	#else
779	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
780	#endif
781	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
782	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
783	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
784	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
785	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
786
787	/* 32-bit nested paging mode: */
788	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
789	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
790	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
791	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
792	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
793	#ifdef VBOX_WITH_64_BITS_GUESTS
794	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
795	#else
796	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
797	#endif
798	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
799	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
800	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
801	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
802	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
803
804	/* PAE nested paging mode: */
805	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
806	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
807	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
808	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
809	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
810	#ifdef VBOX_WITH_64_BITS_GUESTS
811	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
812	#else
813	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
814	#endif
815	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
816	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
817	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
818	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
819	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
820
821	/* AMD64 nested paging mode: */
822	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
823	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
824	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
825	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
826	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
827	#ifdef VBOX_WITH_64_BITS_GUESTS
828	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
829	#else
830	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
831	#endif
832	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
833	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
834	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
835	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
836	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
837
838	/* EPT nested paging mode: */
839	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
840	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
841	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
842	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
843	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
844	#ifdef VBOX_WITH_64_BITS_GUESTS
845	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
846	#else
847	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
848	#endif
849	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
850	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
851	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
852	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
853	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
854
855	/* NONE / NEM: */
856	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
857	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
858	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
859	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
860	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
861	#ifdef VBOX_WITH_64_BITS_GUESTS
862	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
863	#else
864	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
865	#endif
866	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
867	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
868	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
869	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
870	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
871
872
873	#undef PGMMODEDATABTH_ENTRY
874	#undef PGMMODEDATABTH_NULL_ENTRY
875	};
876
877
878	#ifdef IN_RING0
879	/**
880	* #PF Handler.
881	*
882	* @returns VBox status code (appropriate for trap handling and GC return).
883	* @param pVCpu The cross context virtual CPU structure.
884	* @param uErr The trap error code.
885	* @param pRegFrame Trap register frame.
886	* @param pvFault The fault address.
887	*/
888	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
889	{
890	PVM pVM = pVCpu->CTX_SUFF(pVM);
891
892	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
893	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0e, a);
894	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = NULL; } );
895
896
897	# ifdef VBOX_WITH_STATISTICS
898	/*
899	* Error code stats.
900	*/
901	if (uErr & X86_TRAP_PF_US)
902	{
903	if (!(uErr & X86_TRAP_PF_P))
904	{
905	if (uErr & X86_TRAP_PF_RW)
906	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentWrite);
907	else
908	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentRead);
909	}
910	else if (uErr & X86_TRAP_PF_RW)
911	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSWrite);
912	else if (uErr & X86_TRAP_PF_RSVD)
913	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSReserved);
914	else if (uErr & X86_TRAP_PF_ID)
915	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNXE);
916	else
917	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSRead);
918	}
919	else
920	{ /* Supervisor */
921	if (!(uErr & X86_TRAP_PF_P))
922	{
923	if (uErr & X86_TRAP_PF_RW)
924	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentWrite);
925	else
926	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentRead);
927	}
928	else if (uErr & X86_TRAP_PF_RW)
929	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVWrite);
930	else if (uErr & X86_TRAP_PF_ID)
931	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSNXE);
932	else if (uErr & X86_TRAP_PF_RSVD)
933	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVReserved);
934	}
935	# endif /* VBOX_WITH_STATISTICS */
936
937	/*
938	* Call the worker.
939	*/
940	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
941	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
942	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
943	bool fLockTaken = false;
944	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pRegFrame, pvFault, &fLockTaken);
945	if (fLockTaken)
946	{
947	PGM_LOCK_ASSERT_OWNER(pVM);
948	pgmUnlock(pVM);
949	}
950	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
951
952	/*
953	* Return code tweaks.
954	*/
955	if (rc != VINF_SUCCESS)
956	{
957	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
958	rc = VINF_SUCCESS;
959
960	/* Note: hack alert for difficult to reproduce problem. */
961	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
962	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
963	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
964	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
965	{
966	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pRegFrame->rip));
967	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
968	rc = VINF_SUCCESS;
969	}
970	}
971
972	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPF); });
973	STAM_STATS({ if (!pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution))
974	pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Misc; });
975	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0e, pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution), a);
976	return rc;
977	}
978	#endif /* IN_RING0 */
979
980
981	/**
982	* Prefetch a page
983	*
984	* Typically used to sync commonly used pages before entering raw mode
985	* after a CR3 reload.
986	*
987	* @returns VBox status code suitable for scheduling.
988	* @retval VINF_SUCCESS on success.
989	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
990	* @param pVCpu The cross context virtual CPU structure.
991	* @param GCPtrPage Page to invalidate.
992	*/
993	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
994	{
995	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,Prefetch), a);
996
997	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
998	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
999	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
1000	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
1001
1002	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,Prefetch), a);
1003	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1004	return rc;
1005	}
1006
1007
1008	#ifndef PGM_WITHOUT_MAPPINGS
1009	/**
1010	* Gets the mapping corresponding to the specified address (if any).
1011	*
1012	* @returns Pointer to the mapping.
1013	* @returns NULL if not
1014	*
1015	* @param pVM The cross context VM structure.
1016	* @param GCPtr The guest context pointer.
1017	*/
1018	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr)
1019	{
1020	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
1021	while (pMapping)
1022	{
1023	if ((uintptr_t)GCPtr < (uintptr_t)pMapping->GCPtr)
1024	break;
1025	if ((uintptr_t)GCPtr - (uintptr_t)pMapping->GCPtr < pMapping->cb)
1026	return pMapping;
1027	pMapping = pMapping->CTX_SUFF(pNext);
1028	}
1029	return NULL;
1030	}
1031	#endif
1032
1033
1034	/**
1035	* Verifies a range of pages for read or write access
1036	*
1037	* Only checks the guest's page tables
1038	*
1039	* @returns VBox status code.
1040	* @param pVCpu The cross context virtual CPU structure.
1041	* @param Addr Guest virtual address to check
1042	* @param cbSize Access size
1043	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
1044	* @remarks Current not in use.
1045	*/
1046	VMMDECL(int) PGMIsValidAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
1047	{
1048	/*
1049	* Validate input.
1050	*/
1051	if (fAccess & ~(X86_PTE_US \| X86_PTE_RW))
1052	{
1053	AssertMsgFailed(("PGMIsValidAccess: invalid access type %08x\n", fAccess));
1054	return VERR_INVALID_PARAMETER;
1055	}
1056
1057	uint64_t fPage;
1058	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPage, NULL);
1059	if (RT_FAILURE(rc))
1060	{
1061	Log(("PGMIsValidAccess: access violation for %RGv rc=%d\n", Addr, rc));
1062	return VINF_EM_RAW_GUEST_TRAP;
1063	}
1064
1065	/*
1066	* Check if the access would cause a page fault
1067	*
1068	* Note that hypervisor page directories are not present in the guest's tables, so this check
1069	* is sufficient.
1070	*/
1071	bool fWrite = !!(fAccess & X86_PTE_RW);
1072	bool fUser = !!(fAccess & X86_PTE_US);
1073	if ( !(fPage & X86_PTE_P)
1074	\|\| (fWrite && !(fPage & X86_PTE_RW))
1075	\|\| (fUser && !(fPage & X86_PTE_US)) )
1076	{
1077	Log(("PGMIsValidAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPage, fWrite, fUser));
1078	return VINF_EM_RAW_GUEST_TRAP;
1079	}
1080	if ( RT_SUCCESS(rc)
1081	&& PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize))
1082	return PGMIsValidAccess(pVCpu, Addr + PAGE_SIZE, (cbSize > PAGE_SIZE) ? cbSize - PAGE_SIZE : 1, fAccess);
1083	return rc;
1084	}
1085
1086
1087	/**
1088	* Verifies a range of pages for read or write access
1089	*
1090	* Supports handling of pages marked for dirty bit tracking and CSAM
1091	*
1092	* @returns VBox status code.
1093	* @param pVCpu The cross context virtual CPU structure.
1094	* @param Addr Guest virtual address to check
1095	* @param cbSize Access size
1096	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
1097	*/
1098	VMMDECL(int) PGMVerifyAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
1099	{
1100	PVM pVM = pVCpu->CTX_SUFF(pVM);
1101
1102	AssertMsg(!(fAccess & ~(X86_PTE_US \| X86_PTE_RW)), ("PGMVerifyAccess: invalid access type %08x\n", fAccess));
1103
1104	/*
1105	* Get going.
1106	*/
1107	uint64_t fPageGst;
1108	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPageGst, NULL);
1109	if (RT_FAILURE(rc))
1110	{
1111	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", Addr, rc));
1112	return VINF_EM_RAW_GUEST_TRAP;
1113	}
1114
1115	/*
1116	* Check if the access would cause a page fault
1117	*
1118	* Note that hypervisor page directories are not present in the guest's tables, so this check
1119	* is sufficient.
1120	*/
1121	const bool fWrite = !!(fAccess & X86_PTE_RW);
1122	const bool fUser = !!(fAccess & X86_PTE_US);
1123	if ( !(fPageGst & X86_PTE_P)
1124	\|\| (fWrite && !(fPageGst & X86_PTE_RW))
1125	\|\| (fUser && !(fPageGst & X86_PTE_US)) )
1126	{
1127	Log(("PGMVerifyAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPageGst, fWrite, fUser));
1128	return VINF_EM_RAW_GUEST_TRAP;
1129	}
1130
1131	if (!pVM->pgm.s.fNestedPaging)
1132	{
1133	/*
1134	* Next step is to verify if we protected this page for dirty bit tracking or for CSAM scanning
1135	*/
1136	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, NULL, NULL);
1137	if ( rc == VERR_PAGE_NOT_PRESENT
1138	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT)
1139	{
1140	/*
1141	* Page is not present in our page tables.
1142	* Try to sync it!
1143	*/
1144	Assert(X86_TRAP_PF_RW == X86_PTE_RW && X86_TRAP_PF_US == X86_PTE_US);
1145	uint32_t const uErr = fAccess & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
1146	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1147	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1148	AssertReturn(g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
1149	rc = g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage(pVCpu, Addr, fPageGst, uErr);
1150	if (rc != VINF_SUCCESS)
1151	return rc;
1152	}
1153	else
1154	AssertMsg(rc == VINF_SUCCESS, ("PGMShwGetPage %RGv failed with %Rrc\n", Addr, rc));
1155	}
1156
1157	#if 0 /* def VBOX_STRICT; triggers too often now */
1158	/*
1159	* This check is a bit paranoid, but useful.
1160	*/
1161	/* Note! This will assert when writing to monitored pages (a bit annoying actually). */
1162	uint64_t fPageShw;
1163	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, &fPageShw, NULL);
1164	if ( (rc == VERR_PAGE_NOT_PRESENT \|\| RT_FAILURE(rc))
1165	\|\| (fWrite && !(fPageShw & X86_PTE_RW))
1166	\|\| (fUser && !(fPageShw & X86_PTE_US)) )
1167	{
1168	AssertMsgFailed(("Unexpected access violation for %RGv! rc=%Rrc write=%d user=%d\n",
1169	Addr, rc, fWrite && !(fPageShw & X86_PTE_RW), fUser && !(fPageShw & X86_PTE_US)));
1170	return VINF_EM_RAW_GUEST_TRAP;
1171	}
1172	#endif
1173
1174	if ( RT_SUCCESS(rc)
1175	&& ( PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize - 1)
1176	\|\| Addr + cbSize < Addr))
1177	{
1178	/* Don't recursively call PGMVerifyAccess as we might run out of stack. */
1179	for (;;)
1180	{
1181	Addr += PAGE_SIZE;
1182	if (cbSize > PAGE_SIZE)
1183	cbSize -= PAGE_SIZE;
1184	else
1185	cbSize = 1;
1186	rc = PGMVerifyAccess(pVCpu, Addr, 1, fAccess);
1187	if (rc != VINF_SUCCESS)
1188	break;
1189	if (PAGE_ADDRESS(Addr) == PAGE_ADDRESS(Addr + cbSize - 1))
1190	break;
1191	}
1192	}
1193	return rc;
1194	}
1195
1196
1197	/**
1198	* Emulation of the invlpg instruction (HC only actually).
1199	*
1200	* @returns Strict VBox status code, special care required.
1201	* @retval VINF_PGM_SYNC_CR3 - handled.
1202	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1203	* @retval VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
1204	*
1205	* @param pVCpu The cross context virtual CPU structure.
1206	* @param GCPtrPage Page to invalidate.
1207	*
1208	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1209	* safe, but there could be edge cases!
1210	*
1211	* @todo Flush page or page directory only if necessary!
1212	* @todo VBOXSTRICTRC
1213	*/
1214	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1215	{
1216	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1217	int rc;
1218	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1219
1220	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1221
1222	/*
1223	* Call paging mode specific worker.
1224	*/
1225	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage), a);
1226	pgmLock(pVM);
1227
1228	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1229	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), pgmUnlock(pVM), VERR_PGM_MODE_IPE);
1230	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, pgmUnlock(pVM), VERR_PGM_MODE_IPE);
1231	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1232
1233	pgmUnlock(pVM);
1234	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage), a);
1235
1236	#ifdef IN_RING3
1237	/*
1238	* Check if we have a pending update of the CR3 monitoring.
1239	*/
1240	if ( RT_SUCCESS(rc)
1241	&& (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3))
1242	{
1243	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
1244	Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
1245	}
1246	#endif /* IN_RING3 */
1247
1248	/* Ignore all irrelevant error codes. */
1249	if ( rc == VERR_PAGE_NOT_PRESENT
1250	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1251	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1252	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1253	rc = VINF_SUCCESS;
1254
1255	return rc;
1256	}
1257
1258
1259	/**
1260	* Executes an instruction using the interpreter.
1261	*
1262	* @returns VBox status code (appropriate for trap handling and GC return).
1263	* @param pVM The cross context VM structure.
1264	* @param pVCpu The cross context virtual CPU structure.
1265	* @param pRegFrame Register frame.
1266	* @param pvFault Fault address.
1267	*/
1268	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
1269	{
1270	NOREF(pVM);
1271	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, pRegFrame, pvFault);
1272	if (rc == VERR_EM_INTERPRETER)
1273	rc = VINF_EM_RAW_EMULATE_INSTR;
1274	if (rc != VINF_SUCCESS)
1275	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1276	return rc;
1277	}
1278
1279
1280	/**
1281	* Gets effective page information (from the VMM page directory).
1282	*
1283	* @returns VBox status code.
1284	* @param pVCpu The cross context virtual CPU structure.
1285	* @param GCPtr Guest Context virtual address of the page.
1286	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1287	* @param pHCPhys Where to store the HC physical address of the page.
1288	* This is page aligned.
1289	* @remark You should use PGMMapGetPage() for pages in a mapping.
1290	*/
1291	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1292	{
1293	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1294	pgmLock(pVM);
1295
1296	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1297	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1298	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1299	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1300
1301	pgmUnlock(pVM);
1302	return rc;
1303	}
1304
1305
1306	/**
1307	* Modify page flags for a range of pages in the shadow context.
1308	*
1309	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1310	*
1311	* @returns VBox status code.
1312	* @param pVCpu The cross context virtual CPU structure.
1313	* @param GCPtr Virtual address of the first page in the range.
1314	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1315	* @param fMask The AND mask - page flags X86_PTE_*.
1316	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1317	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1318	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1319	*/
1320	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1321	{
1322	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1323	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1324
1325	GCPtr &= PAGE_BASE_GC_MASK; /** @todo this ain't necessary, right... */
1326
1327	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1328	pgmLock(pVM);
1329
1330	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1331	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1332	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1333	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, PAGE_SIZE, fFlags, fMask, fOpFlags);
1334
1335	pgmUnlock(pVM);
1336	return rc;
1337	}
1338
1339
1340	/**
1341	* Changing the page flags for a single page in the shadow page tables so as to
1342	* make it read-only.
1343	*
1344	* @returns VBox status code.
1345	* @param pVCpu The cross context virtual CPU structure.
1346	* @param GCPtr Virtual address of the first page in the range.
1347	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1348	*/
1349	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1350	{
1351	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1352	}
1353
1354
1355	/**
1356	* Changing the page flags for a single page in the shadow page tables so as to
1357	* make it writable.
1358	*
1359	* The call must know with 101% certainty that the guest page tables maps this
1360	* as writable too. This function will deal shared, zero and write monitored
1361	* pages.
1362	*
1363	* @returns VBox status code.
1364	* @param pVCpu The cross context virtual CPU structure.
1365	* @param GCPtr Virtual address of the first page in the range.
1366	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1367	*/
1368	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1369	{
1370	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1371	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1372	return VINF_SUCCESS;
1373	}
1374
1375
1376	/**
1377	* Changing the page flags for a single page in the shadow page tables so as to
1378	* make it not present.
1379	*
1380	* @returns VBox status code.
1381	* @param pVCpu The cross context virtual CPU structure.
1382	* @param GCPtr Virtual address of the first page in the range.
1383	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1384	*/
1385	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1386	{
1387	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1388	}
1389
1390
1391	/**
1392	* Changing the page flags for a single page in the shadow page tables so as to
1393	* make it supervisor and writable.
1394	*
1395	* This if for dealing with CR0.WP=0 and readonly user pages.
1396	*
1397	* @returns VBox status code.
1398	* @param pVCpu The cross context virtual CPU structure.
1399	* @param GCPtr Virtual address of the first page in the range.
1400	* @param fBigPage Whether or not this is a big page. If it is, we have to
1401	* change the shadow PDE as well. If it isn't, the caller
1402	* has checked that the shadow PDE doesn't need changing.
1403	* We ASSUME 4KB pages backing the big page here!
1404	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1405	*/
1406	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1407	{
1408	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1409	if (rc == VINF_SUCCESS && fBigPage)
1410	{
1411	/* this is a bit ugly... */
1412	switch (pVCpu->pgm.s.enmShadowMode)
1413	{
1414	case PGMMODE_32_BIT:
1415	{
1416	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1417	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1418	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1419	pPde->n.u1Write = 1;
1420	Log(("-> PDE=%#llx (32)\n", pPde->u));
1421	break;
1422	}
1423	case PGMMODE_PAE:
1424	case PGMMODE_PAE_NX:
1425	{
1426	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1427	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1428	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1429	pPde->n.u1Write = 1;
1430	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1431	break;
1432	}
1433	default:
1434	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1435	}
1436	}
1437	return rc;
1438	}
1439
1440
1441	/**
1442	* Gets the shadow page directory for the specified address, PAE.
1443	*
1444	* @returns Pointer to the shadow PD.
1445	* @param pVCpu The cross context virtual CPU structure.
1446	* @param GCPtr The address.
1447	* @param uGstPdpe Guest PDPT entry. Valid.
1448	* @param ppPD Receives address of page directory
1449	*/
1450	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1451	{
1452	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1453	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1454	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1455	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1456	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1457	PPGMPOOLPAGE pShwPage;
1458	int rc;
1459
1460	PGM_LOCK_ASSERT_OWNER(pVM);
1461
1462	/* Allocate page directory if not present. */
1463	if ( !pPdpe->n.u1Present
1464	&& !(pPdpe->u & X86_PDPE_PG_MASK))
1465	{
1466	RTGCPTR64 GCPdPt;
1467	PGMPOOLKIND enmKind;
1468
1469	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1470	{
1471	/* AMD-V nested paging or real/protected mode without paging. */
1472	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1473	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1474	}
1475	else
1476	{
1477	if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1478	{
1479	if (!(uGstPdpe & X86_PDPE_P))
1480	{
1481	/* PD not present; guest must reload CR3 to change it.
1482	* No need to monitor anything in this case.
1483	*/
1484	Assert(VM_IS_RAW_MODE_ENABLED(pVM));
1485
1486	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1487	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1488	uGstPdpe \|= X86_PDPE_P;
1489	}
1490	else
1491	{
1492	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1493	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1494	}
1495	}
1496	else
1497	{
1498	GCPdPt = CPUMGetGuestCR3(pVCpu);
1499	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1500	}
1501	}
1502
1503	/* Create a reference back to the PDPT by using the index in its shadow page. */
1504	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1505	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1506	&pShwPage);
1507	AssertRCReturn(rc, rc);
1508
1509	/* The PD was cached or created; hook it up now. */
1510	pPdpe->u \|= pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A));
1511	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1512	}
1513	else
1514	{
1515	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
1516	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1517	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1518
1519	pgmPoolCacheUsed(pPool, pShwPage);
1520	}
1521	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1522	return VINF_SUCCESS;
1523	}
1524
1525
1526	/**
1527	* Gets the pointer to the shadow page directory entry for an address, PAE.
1528	*
1529	* @returns Pointer to the PDE.
1530	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1531	* @param GCPtr The address.
1532	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1533	*/
1534	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1535	{
1536	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1537	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1538	PVM pVM = pVCpu->CTX_SUFF(pVM);
1539
1540	PGM_LOCK_ASSERT_OWNER(pVM);
1541
1542	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1543	if (!pPdpt->a[iPdPt].n.u1Present)
1544	{
1545	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, pPdpt->a[iPdPt].u));
1546	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1547	}
1548	AssertMsg(pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1549
1550	/* Fetch the pgm pool shadow descriptor. */
1551	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
1552	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1553
1554	*ppShwPde = pShwPde;
1555	return VINF_SUCCESS;
1556	}
1557
1558
1559	/**
1560	* Syncs the SHADOW page directory pointer for the specified address.
1561	*
1562	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1563	*
1564	* The caller is responsible for making sure the guest has a valid PD before
1565	* calling this function.
1566	*
1567	* @returns VBox status code.
1568	* @param pVCpu The cross context virtual CPU structure.
1569	* @param GCPtr The address.
1570	* @param uGstPml4e Guest PML4 entry (valid).
1571	* @param uGstPdpe Guest PDPT entry (valid).
1572	* @param ppPD Receives address of page directory
1573	*/
1574	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1575	{
1576	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1577	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1578	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1579	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1580	bool fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1581	PPGMPOOLPAGE pShwPage;
1582	int rc;
1583
1584	PGM_LOCK_ASSERT_OWNER(pVM);
1585
1586	/* Allocate page directory pointer table if not present. */
1587	if ( !pPml4e->n.u1Present
1588	&& !(pPml4e->u & X86_PML4E_PG_MASK))
1589	{
1590	RTGCPTR64 GCPml4;
1591	PGMPOOLKIND enmKind;
1592
1593	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1594
1595	if (fNestedPagingOrNoGstPaging)
1596	{
1597	/* AMD-V nested paging or real/protected mode without paging */
1598	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT;
1599	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1600	}
1601	else
1602	{
1603	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1604	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1605	}
1606
1607	/* Create a reference back to the PDPT by using the index in its shadow page. */
1608	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1609	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1610	&pShwPage);
1611	AssertRCReturn(rc, rc);
1612	}
1613	else
1614	{
1615	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
1616	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1617
1618	pgmPoolCacheUsed(pPool, pShwPage);
1619	}
1620	/* The PDPT was cached or created; hook it up now. */
1621	pPml4e->u \|= pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask);
1622
1623	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1624	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1625	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1626
1627	/* Allocate page directory if not present. */
1628	if ( !pPdpe->n.u1Present
1629	&& !(pPdpe->u & X86_PDPE_PG_MASK))
1630	{
1631	RTGCPTR64 GCPdPt;
1632	PGMPOOLKIND enmKind;
1633
1634	if (fNestedPagingOrNoGstPaging)
1635	{
1636	/* AMD-V nested paging or real/protected mode without paging */
1637	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1638	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1639	}
1640	else
1641	{
1642	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1643	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1644	}
1645
1646	/* Create a reference back to the PDPT by using the index in its shadow page. */
1647	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1648	pShwPage->idx, iPdPt, false /fLockPage/,
1649	&pShwPage);
1650	AssertRCReturn(rc, rc);
1651	}
1652	else
1653	{
1654	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
1655	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1656
1657	pgmPoolCacheUsed(pPool, pShwPage);
1658	}
1659	/* The PD was cached or created; hook it up now. */
1660	pPdpe->u \|= pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask);
1661
1662	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1663	return VINF_SUCCESS;
1664	}
1665
1666
1667	/**
1668	* Gets the SHADOW page directory pointer for the specified address (long mode).
1669	*
1670	* @returns VBox status code.
1671	* @param pVCpu The cross context virtual CPU structure.
1672	* @param GCPtr The address.
1673	* @param ppPml4e Receives the address of the page map level 4 entry.
1674	* @param ppPdpt Receives the address of the page directory pointer table.
1675	* @param ppPD Receives the address of the page directory.
1676	*/
1677	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1678	{
1679	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1680	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1681
1682	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1683
1684	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1685	if (ppPml4e)
1686	*ppPml4e = (PX86PML4E)pPml4e;
1687
1688	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, pPml4e->u));
1689
1690	if (!pPml4e->n.u1Present)
1691	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1692
1693	PVM pVM = pVCpu->CTX_SUFF(pVM);
1694	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1695	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
1696	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1697
1698	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1699	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1700	if (!pPdpt->a[iPdPt].n.u1Present)
1701	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1702
1703	pShwPage = pgmPoolGetPage(pPool, pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
1704	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1705
1706	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1707	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1708	return VINF_SUCCESS;
1709	}
1710
1711
1712	/**
1713	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1714	* backing pages in case the PDPT or PML4 entry is missing.
1715	*
1716	* @returns VBox status code.
1717	* @param pVCpu The cross context virtual CPU structure.
1718	* @param GCPtr The address.
1719	* @param ppPdpt Receives address of pdpt
1720	* @param ppPD Receives address of page directory
1721	*/
1722	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1723	{
1724	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1725	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1726	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1727	PEPTPML4 pPml4;
1728	PEPTPML4E pPml4e;
1729	PPGMPOOLPAGE pShwPage;
1730	int rc;
1731
1732	Assert(pVM->pgm.s.fNestedPaging);
1733	PGM_LOCK_ASSERT_OWNER(pVM);
1734
1735	pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1736	Assert(pPml4);
1737
1738	/* Allocate page directory pointer table if not present. */
1739	pPml4e = &pPml4->a[iPml4];
1740	if ( !pPml4e->n.u1Present
1741	&& !(pPml4e->u & EPT_PML4E_PG_MASK))
1742	{
1743	Assert(!(pPml4e->u & EPT_PML4E_PG_MASK));
1744	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1745
1746	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1747	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1748	&pShwPage);
1749	AssertRCReturn(rc, rc);
1750	}
1751	else
1752	{
1753	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1754	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1755
1756	pgmPoolCacheUsed(pPool, pShwPage);
1757	}
1758	/* The PDPT was cached or created; hook it up now and fill with the default value. */
1759	pPml4e->u = pShwPage->Core.Key;
1760	pPml4e->n.u1Present = 1;
1761	pPml4e->n.u1Write = 1;
1762	pPml4e->n.u1Execute = 1;
1763
1764	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1765	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1766	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1767
1768	if (ppPdpt)
1769	*ppPdpt = pPdpt;
1770
1771	/* Allocate page directory if not present. */
1772	if ( !pPdpe->n.u1Present
1773	&& !(pPdpe->u & EPT_PDPTE_PG_MASK))
1774	{
1775	RTGCPTR64 GCPdPt = (RTGCPTR64)iPdPt << EPT_PDPT_SHIFT;
1776	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1777	pShwPage->idx, iPdPt, false /fLockPage/,
1778	&pShwPage);
1779	AssertRCReturn(rc, rc);
1780	}
1781	else
1782	{
1783	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1784	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1785
1786	pgmPoolCacheUsed(pPool, pShwPage);
1787	}
1788	/* The PD was cached or created; hook it up now and fill with the default value. */
1789	pPdpe->u = pShwPage->Core.Key;
1790	pPdpe->n.u1Present = 1;
1791	pPdpe->n.u1Write = 1;
1792	pPdpe->n.u1Execute = 1;
1793
1794	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1795	return VINF_SUCCESS;
1796	}
1797
1798
1799	#ifdef IN_RING0
1800	/**
1801	* Synchronizes a range of nested page table entries.
1802	*
1803	* The caller must own the PGM lock.
1804	*
1805	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1806	* @param GCPhys Where to start.
1807	* @param cPages How many pages which entries should be synced.
1808	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1809	* host paging mode for AMD-V).
1810	*/
1811	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1812	{
1813	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1814
1815	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1816	int rc;
1817	switch (enmShwPagingMode)
1818	{
1819	case PGMMODE_32_BIT:
1820	{
1821	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1822	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1823	break;
1824	}
1825
1826	case PGMMODE_PAE:
1827	case PGMMODE_PAE_NX:
1828	{
1829	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1830	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1831	break;
1832	}
1833
1834	case PGMMODE_AMD64:
1835	case PGMMODE_AMD64_NX:
1836	{
1837	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1838	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1839	break;
1840	}
1841
1842	case PGMMODE_EPT:
1843	{
1844	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1845	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1846	break;
1847	}
1848
1849	default:
1850	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1851	}
1852	return rc;
1853	}
1854	#endif /* IN_RING0 */
1855
1856
1857	/**
1858	* Gets effective Guest OS page information.
1859	*
1860	* When GCPtr is in a big page, the function will return as if it was a normal
1861	* 4KB page. If the need for distinguishing between big and normal page becomes
1862	* necessary at a later point, a PGMGstGetPage() will be created for that
1863	* purpose.
1864	*
1865	* @returns VBox status code.
1866	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1867	* @param GCPtr Guest Context virtual address of the page.
1868	* @param pfFlags Where to store the flags. These are X86_PTE_*, even for big pages.
1869	* @param pGCPhys Where to store the GC physical address of the page.
1870	* This is page aligned. The fact that the
1871	*/
1872	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys)
1873	{
1874	VMCPU_ASSERT_EMT(pVCpu);
1875	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1876	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1877	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1878	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pfFlags, pGCPhys);
1879	}
1880
1881
1882	/**
1883	* Performs a guest page table walk.
1884	*
1885	* The guest should be in paged protect mode or long mode when making a call to
1886	* this function.
1887	*
1888	* @returns VBox status code.
1889	* @retval VINF_SUCCESS on success.
1890	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1891	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1892	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1893	*
1894	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1895	* @param GCPtr The guest virtual address to walk by.
1896	* @param pWalk Where to return the walk result. This is valid for some
1897	* error codes as well.
1898	*/
1899	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
1900	{
1901	VMCPU_ASSERT_EMT(pVCpu);
1902	switch (pVCpu->pgm.s.enmGuestMode)
1903	{
1904	case PGMMODE_32_BIT:
1905	pWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
1906	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, &pWalk->u.Legacy);
1907
1908	case PGMMODE_PAE:
1909	case PGMMODE_PAE_NX:
1910	pWalk->enmType = PGMPTWALKGSTTYPE_PAE;
1911	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, &pWalk->u.Pae);
1912
1913	case PGMMODE_AMD64:
1914	case PGMMODE_AMD64_NX:
1915	pWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
1916	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, &pWalk->u.Amd64);
1917
1918	case PGMMODE_REAL:
1919	case PGMMODE_PROTECTED:
1920	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1921	return VERR_PGM_NOT_USED_IN_MODE;
1922
1923	case PGMMODE_NESTED_32BIT:
1924	case PGMMODE_NESTED_PAE:
1925	case PGMMODE_NESTED_AMD64:
1926	case PGMMODE_EPT:
1927	default:
1928	AssertFailed();
1929	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1930	return VERR_PGM_NOT_USED_IN_MODE;
1931	}
1932	}
1933
1934
1935	/**
1936	* Tries to continue the previous walk.
1937	*
1938	* @note Requires the caller to hold the PGM lock from the first
1939	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
1940	* we cannot use the pointers.
1941	*
1942	* @returns VBox status code.
1943	* @retval VINF_SUCCESS on success.
1944	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1945	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1946	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1947	*
1948	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1949	* @param GCPtr The guest virtual address to walk by.
1950	* @param pWalk Pointer to the previous walk result and where to return
1951	* the result of this walk. This is valid for some error
1952	* codes as well.
1953	*/
1954	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
1955	{
1956	/*
1957	* We can only handle successfully walks.
1958	* We also limit ourselves to the next page.
1959	*/
1960	if ( pWalk->u.Core.fSucceeded
1961	&& GCPtr - pWalk->u.Core.GCPtr == PAGE_SIZE)
1962	{
1963	Assert(pWalk->u.Core.uLevel == 0);
1964	if (pWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
1965	{
1966	/*
1967	* AMD64
1968	*/
1969	if (!pWalk->u.Core.fGigantPage && !pWalk->u.Core.fBigPage)
1970	{
1971	/*
1972	* We fall back to full walk if the PDE table changes, if any
1973	* reserved bits are set, or if the effective page access changes.
1974	*/
1975	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
1976	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
1977	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
1978	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
1979
1980	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PD_PAE_SHIFT))
1981	{
1982	if (pWalk->u.Amd64.pPte)
1983	{
1984	X86PTEPAE Pte;
1985	Pte.u = pWalk->u.Amd64.pPte[1].u;
1986	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
1987	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
1988	{
1989
1990	pWalk->u.Core.GCPtr = GCPtr;
1991	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
1992	pWalk->u.Amd64.Pte.u = Pte.u;
1993	pWalk->u.Amd64.pPte++;
1994	return VINF_SUCCESS;
1995	}
1996	}
1997	}
1998	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PDPT_SHIFT))
1999	{
2000	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2001	if (pWalk->u.Amd64.pPde)
2002	{
2003	X86PDEPAE Pde;
2004	Pde.u = pWalk->u.Amd64.pPde[1].u;
2005	if ( (Pde.u & fPdeSame) == (pWalk->u.Amd64.Pde.u & fPdeSame)
2006	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2007	{
2008	/* Get the new PTE and check out the first entry. */
2009	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2010	&pWalk->u.Amd64.pPt);
2011	if (RT_SUCCESS(rc))
2012	{
2013	pWalk->u.Amd64.pPte = &pWalk->u.Amd64.pPt->a[0];
2014	X86PTEPAE Pte;
2015	Pte.u = pWalk->u.Amd64.pPte->u;
2016	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
2017	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2018	{
2019	pWalk->u.Core.GCPtr = GCPtr;
2020	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2021	pWalk->u.Amd64.Pte.u = Pte.u;
2022	pWalk->u.Amd64.Pde.u = Pde.u;
2023	pWalk->u.Amd64.pPde++;
2024	return VINF_SUCCESS;
2025	}
2026	}
2027	}
2028	}
2029	}
2030	}
2031	else if (!pWalk->u.Core.fGigantPage)
2032	{
2033	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_2M_BASE_MASK))
2034	{
2035	pWalk->u.Core.GCPtr = GCPtr;
2036	pWalk->u.Core.GCPhys += PAGE_SIZE;
2037	return VINF_SUCCESS;
2038	}
2039	}
2040	else
2041	{
2042	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_1G_BASE_MASK))
2043	{
2044	pWalk->u.Core.GCPtr = GCPtr;
2045	pWalk->u.Core.GCPhys += PAGE_SIZE;
2046	return VINF_SUCCESS;
2047	}
2048	}
2049	}
2050	}
2051	/* Case we don't handle. Do full walk. */
2052	return pgmGstPtWalk(pVCpu, GCPtr, pWalk);
2053	}
2054
2055
2056	/**
2057	* Checks if the page is present.
2058	*
2059	* @returns true if the page is present.
2060	* @returns false if the page is not present.
2061	* @param pVCpu The cross context virtual CPU structure.
2062	* @param GCPtr Address within the page.
2063	*/
2064	VMMDECL(bool) PGMGstIsPagePresent(PVMCPUCC pVCpu, RTGCPTR GCPtr)
2065	{
2066	VMCPU_ASSERT_EMT(pVCpu);
2067	int rc = PGMGstGetPage(pVCpu, GCPtr, NULL, NULL);
2068	return RT_SUCCESS(rc);
2069	}
2070
2071
2072	/**
2073	* Sets (replaces) the page flags for a range of pages in the guest's tables.
2074	*
2075	* @returns VBox status code.
2076	* @param pVCpu The cross context virtual CPU structure.
2077	* @param GCPtr The address of the first page.
2078	* @param cb The size of the range in bytes.
2079	* @param fFlags Page flags X86_PTE_*, excluding the page mask of course.
2080	*/
2081	VMMDECL(int) PGMGstSetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags)
2082	{
2083	VMCPU_ASSERT_EMT(pVCpu);
2084	return PGMGstModifyPage(pVCpu, GCPtr, cb, fFlags, 0);
2085	}
2086
2087
2088	/**
2089	* Modify page flags for a range of pages in the guest's tables
2090	*
2091	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2092	*
2093	* @returns VBox status code.
2094	* @param pVCpu The cross context virtual CPU structure.
2095	* @param GCPtr Virtual address of the first page in the range.
2096	* @param cb Size (in bytes) of the range to apply the modification to.
2097	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2098	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2099	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2100	*/
2101	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2102	{
2103	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,GstModifyPage), a);
2104	VMCPU_ASSERT_EMT(pVCpu);
2105
2106	/*
2107	* Validate input.
2108	*/
2109	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2110	Assert(cb);
2111
2112	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2113
2114	/*
2115	* Adjust input.
2116	*/
2117	cb += GCPtr & PAGE_OFFSET_MASK;
2118	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
2119	GCPtr = (GCPtr & PAGE_BASE_GC_MASK);
2120
2121	/*
2122	* Call worker.
2123	*/
2124	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2125	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2126	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2127	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2128
2129	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,GstModifyPage), a);
2130	return rc;
2131	}
2132
2133
2134	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2135
2136	/**
2137	* Performs the lazy mapping of the 32-bit guest PD.
2138	*
2139	* @returns VBox status code.
2140	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2141	* @param ppPd Where to return the pointer to the mapping. This is
2142	* always set.
2143	*/
2144	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2145	{
2146	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2147	pgmLock(pVM);
2148
2149	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2150
2151	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAGE_MASK;
2152	PPGMPAGE pPage;
2153	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2154	if (RT_SUCCESS(rc))
2155	{
2156	# ifdef VBOX_WITH_RAM_IN_KERNEL
2157	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2158	if (RT_SUCCESS(rc))
2159	{
2160	# ifdef IN_RING3
2161	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2162	pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2163	# else
2164	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2165	pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2166	# endif
2167	pgmUnlock(pVM);
2168	return VINF_SUCCESS;
2169	}
2170	# else
2171	RTHCPTR HCPtrGuestCR3;
2172	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
2173	if (RT_SUCCESS(rc))
2174	{
2175	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
2176	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2177	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
2178	# endif
2179	*ppPd = (PX86PD)HCPtrGuestCR3;
2180
2181	pgmUnlock(pVM);
2182	return VINF_SUCCESS;
2183	}
2184	# endif
2185	AssertRC(rc);
2186	}
2187	pgmUnlock(pVM);
2188
2189	*ppPd = NULL;
2190	return rc;
2191	}
2192
2193
2194	/**
2195	* Performs the lazy mapping of the PAE guest PDPT.
2196	*
2197	* @returns VBox status code.
2198	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2199	* @param ppPdpt Where to return the pointer to the mapping. This is
2200	* always set.
2201	*/
2202	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2203	{
2204	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2205	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2206	pgmLock(pVM);
2207
2208	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAE_PAGE_MASK;
2209	PPGMPAGE pPage;
2210	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2211	if (RT_SUCCESS(rc))
2212	{
2213	# ifdef VBOX_WITH_RAM_IN_KERNEL
2214	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2215	if (RT_SUCCESS(rc))
2216	{
2217	# ifdef IN_RING3
2218	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2219	pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2220	# else
2221	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2222	pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2223	# endif
2224	pgmUnlock(pVM);
2225	return VINF_SUCCESS;
2226	}
2227	# else
2228	RTHCPTR HCPtrGuestCR3;
2229	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
2230	if (RT_SUCCESS(rc))
2231	{
2232	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
2233	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2234	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
2235	# endif
2236	*ppPdpt = (PX86PDPT)HCPtrGuestCR3;
2237
2238	pgmUnlock(pVM);
2239	return VINF_SUCCESS;
2240	}
2241	# endif
2242	AssertRC(rc);
2243	}
2244
2245	pgmUnlock(pVM);
2246	*ppPdpt = NULL;
2247	return rc;
2248	}
2249
2250
2251	/**
2252	* Performs the lazy mapping / updating of a PAE guest PD.
2253	*
2254	* @returns Pointer to the mapping.
2255	* @returns VBox status code.
2256	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2257	* @param iPdpt Which PD entry to map (0..3).
2258	* @param ppPd Where to return the pointer to the mapping. This is
2259	* always set.
2260	*/
2261	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2262	{
2263	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2264	pgmLock(pVM);
2265
2266	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2267	Assert(pGuestPDPT);
2268	Assert(pGuestPDPT->a[iPdpt].n.u1Present);
2269	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2270	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2271
2272	PPGMPAGE pPage;
2273	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2274	if (RT_SUCCESS(rc))
2275	{
2276	# ifdef VBOX_WITH_RAM_IN_KERNEL
2277	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2278	AssertRC(rc);
2279	if (RT_SUCCESS(rc))
2280	{
2281	# ifdef IN_RING3
2282	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2283	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2284	# else
2285	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2286	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2287	# endif
2288	if (fChanged)
2289	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2290	pgmUnlock(pVM);
2291	return VINF_SUCCESS;
2292	}
2293	# else
2294	RTHCPTR HCPtr = NIL_RTHCPTR;
2295	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2296	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, &HCPtr);
2297	AssertRC(rc);
2298	# endif
2299	if (RT_SUCCESS(rc))
2300	{
2301	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = (R3PTRTYPE(PX86PDPAE))HCPtr;
2302	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2303	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = (R0PTRTYPE(PX86PDPAE))HCPtr;
2304	# endif
2305	if (fChanged)
2306	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2307
2308	*ppPd = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
2309	pgmUnlock(pVM);
2310	return VINF_SUCCESS;
2311	}
2312	# endif
2313	}
2314
2315	/* Invalid page or some failure, invalidate the entry. */
2316	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2317	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2318	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2319	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2320	# endif
2321
2322	pgmUnlock(pVM);
2323	return rc;
2324	}
2325
2326
2327	/**
2328	* Performs the lazy mapping of the 32-bit guest PD.
2329	*
2330	* @returns VBox status code.
2331	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2332	* @param ppPml4 Where to return the pointer to the mapping. This will
2333	* always be set.
2334	*/
2335	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2336	{
2337	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2338	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2339	pgmLock(pVM);
2340
2341	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_AMD64_PAGE_MASK;
2342	PPGMPAGE pPage;
2343	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2344	if (RT_SUCCESS(rc))
2345	{
2346	# ifdef VBOX_WITH_RAM_IN_KERNEL
2347	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2348	if (RT_SUCCESS(rc))
2349	{
2350	# ifdef IN_RING3
2351	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2352	pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2353	# else
2354	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2355	pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2356	# endif
2357	pgmUnlock(pVM);
2358	return VINF_SUCCESS;
2359	}
2360	# else
2361	RTHCPTR HCPtrGuestCR3;
2362	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
2363	if (RT_SUCCESS(rc))
2364	{
2365	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
2366	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2367	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
2368	# endif
2369	*ppPml4 = (PX86PML4)HCPtrGuestCR3;
2370
2371	pgmUnlock(pVM);
2372	return VINF_SUCCESS;
2373	}
2374	# endif
2375	}
2376
2377	pgmUnlock(pVM);
2378	*ppPml4 = NULL;
2379	return rc;
2380	}
2381
2382	#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
2383
2384
2385	/**
2386	* Gets the PAE PDPEs values cached by the CPU.
2387	*
2388	* @returns VBox status code.
2389	* @param pVCpu The cross context virtual CPU structure.
2390	* @param paPdpes Where to return the four PDPEs. The array
2391	* pointed to must have 4 entries.
2392	*/
2393	VMM_INT_DECL(int) PGMGstGetPaePdpes(PVMCPUCC pVCpu, PX86PDPE paPdpes)
2394	{
2395	Assert(pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2396
2397	paPdpes[0] = pVCpu->pgm.s.aGstPaePdpeRegs[0];
2398	paPdpes[1] = pVCpu->pgm.s.aGstPaePdpeRegs[1];
2399	paPdpes[2] = pVCpu->pgm.s.aGstPaePdpeRegs[2];
2400	paPdpes[3] = pVCpu->pgm.s.aGstPaePdpeRegs[3];
2401	return VINF_SUCCESS;
2402	}
2403
2404
2405	/**
2406	* Sets the PAE PDPEs values cached by the CPU.
2407	*
2408	* @remarks This must be called AFTER PGMUpdateCR3.
2409	*
2410	* @param pVCpu The cross context virtual CPU structure.
2411	* @param paPdpes The four PDPE values. The array pointed to must
2412	* have exactly 4 entries.
2413	*
2414	* @remarks No-long-jump zone!!!
2415	*/
2416	VMM_INT_DECL(void) PGMGstUpdatePaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPdpes)
2417	{
2418	Assert(pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2419
2420	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGstPaePdpeRegs); i++)
2421	{
2422	if (pVCpu->pgm.s.aGstPaePdpeRegs[i].u != paPdpes[i].u)
2423	{
2424	pVCpu->pgm.s.aGstPaePdpeRegs[i] = paPdpes[i];
2425
2426	/* Force lazy remapping if it changed in any way. */
2427	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2428	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2429	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2430	#endif
2431	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2432	}
2433	}
2434
2435	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
2436	}
2437
2438
2439	/**
2440	* Gets the current CR3 register value for the shadow memory context.
2441	* @returns CR3 value.
2442	* @param pVCpu The cross context virtual CPU structure.
2443	*/
2444	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2445	{
2446	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2447	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2448	return pPoolPage->Core.Key;
2449	}
2450
2451
2452	/**
2453	* Performs and schedules necessary updates following a CR3 load or reload.
2454	*
2455	* This will normally involve mapping the guest PD or nPDPT
2456	*
2457	* @returns VBox status code.
2458	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2459	* safely be ignored and overridden since the FF will be set too then.
2460	* @param pVCpu The cross context virtual CPU structure.
2461	* @param cr3 The new cr3.
2462	* @param fGlobal Indicates whether this is a global flush or not.
2463	*/
2464	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
2465	{
2466	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLB), a);
2467	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2468
2469	VMCPU_ASSERT_EMT(pVCpu);
2470
2471	/*
2472	* Always flag the necessary updates; necessary for hardware acceleration
2473	*/
2474	/** @todo optimize this, it shouldn't always be necessary. */
2475	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2476	if (fGlobal)
2477	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2478	LogFlow(("PGMFlushTLB: cr3=%RX64 OldCr3=%RX64 fGlobal=%d\n", cr3, pVCpu->pgm.s.GCPhysCR3, fGlobal));
2479
2480	/*
2481	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2482	*/
2483	int rc = VINF_SUCCESS;
2484	RTGCPHYS GCPhysCR3;
2485	switch (pVCpu->pgm.s.enmGuestMode)
2486	{
2487	case PGMMODE_PAE:
2488	case PGMMODE_PAE_NX:
2489	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
2490	break;
2491	case PGMMODE_AMD64:
2492	case PGMMODE_AMD64_NX:
2493	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
2494	break;
2495	default:
2496	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
2497	break;
2498	}
2499	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2500
2501	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2502	if (GCPhysOldCR3 != GCPhysCR3)
2503	{
2504	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2505	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2506	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2507
2508	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2509	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2510	if (RT_LIKELY(rc == VINF_SUCCESS))
2511	{
2512	if (pgmMapAreMappingsFloating(pVM))
2513	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2514	}
2515	else
2516	{
2517	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2518	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2519	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2520	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2521	if (pgmMapAreMappingsFloating(pVM))
2522	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MONITOR_CR3;
2523	}
2524
2525	if (fGlobal)
2526	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2527	else
2528	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBNewCR3));
2529	}
2530	else
2531	{
2532	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2533	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2534	if (pPool->cDirtyPages)
2535	{
2536	pgmLock(pVM);
2537	pgmPoolResetDirtyPages(pVM);
2538	pgmUnlock(pVM);
2539	}
2540	#endif
2541	/*
2542	* Check if we have a pending update of the CR3 monitoring.
2543	*/
2544	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
2545	{
2546	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2547	Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
2548	}
2549	if (fGlobal)
2550	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2551	else
2552	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBSameCR3));
2553	}
2554
2555	IEMTlbInvalidateAll(pVCpu, false /fVmm/);
2556	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLB), a);
2557	return rc;
2558	}
2559
2560
2561	/**
2562	* Performs and schedules necessary updates following a CR3 load or reload when
2563	* using nested or extended paging.
2564	*
2565	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2566	* TLB and triggering a SyncCR3.
2567	*
2568	* This will normally involve mapping the guest PD or nPDPT
2569	*
2570	* @returns VBox status code.
2571	* @retval VINF_SUCCESS.
2572	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2573	* paging modes). This can safely be ignored and overridden since the
2574	* FF will be set too then.
2575	* @param pVCpu The cross context virtual CPU structure.
2576	* @param cr3 The new cr3.
2577	*/
2578	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
2579	{
2580	VMCPU_ASSERT_EMT(pVCpu);
2581	LogFlow(("PGMUpdateCR3: cr3=%RX64 OldCr3=%RX64\n", cr3, pVCpu->pgm.s.GCPhysCR3));
2582
2583	/* We assume we're only called in nested paging mode. */
2584	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2585	Assert(!pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
2586	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3));
2587
2588	/*
2589	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2590	*/
2591	int rc = VINF_SUCCESS;
2592	RTGCPHYS GCPhysCR3;
2593	switch (pVCpu->pgm.s.enmGuestMode)
2594	{
2595	case PGMMODE_PAE:
2596	case PGMMODE_PAE_NX:
2597	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
2598	break;
2599	case PGMMODE_AMD64:
2600	case PGMMODE_AMD64_NX:
2601	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
2602	break;
2603	default:
2604	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
2605	break;
2606	}
2607	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2608
2609	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2610	{
2611	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2612	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2613	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2614
2615	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2616	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2617
2618	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2619	}
2620
2621	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2622	return rc;
2623	}
2624
2625
2626	/**
2627	* Synchronize the paging structures.
2628	*
2629	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2630	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2631	* in several places, most importantly whenever the CR3 is loaded.
2632	*
2633	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2634	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2635	* the VMM into guest context.
2636	* @param pVCpu The cross context virtual CPU structure.
2637	* @param cr0 Guest context CR0 register
2638	* @param cr3 Guest context CR3 register
2639	* @param cr4 Guest context CR4 register
2640	* @param fGlobal Including global page directories or not
2641	*/
2642	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2643	{
2644	int rc;
2645
2646	VMCPU_ASSERT_EMT(pVCpu);
2647
2648	/*
2649	* The pool may have pending stuff and even require a return to ring-3 to
2650	* clear the whole thing.
2651	*/
2652	rc = pgmPoolSyncCR3(pVCpu);
2653	if (rc != VINF_SUCCESS)
2654	return rc;
2655
2656	/*
2657	* We might be called when we shouldn't.
2658	*
2659	* The mode switching will ensure that the PD is resynced after every mode
2660	* switch. So, if we find ourselves here when in protected or real mode
2661	* we can safely clear the FF and return immediately.
2662	*/
2663	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2664	{
2665	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2666	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2667	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2668	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2669	return VINF_SUCCESS;
2670	}
2671
2672	/* If global pages are not supported, then all flushes are global. */
2673	if (!(cr4 & X86_CR4_PGE))
2674	fGlobal = true;
2675	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2676	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2677
2678	/*
2679	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2680	* This should be done before SyncCR3.
2681	*/
2682	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2683	{
2684	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2685
2686	RTGCPHYS GCPhysCR3Old = pVCpu->pgm.s.GCPhysCR3; NOREF(GCPhysCR3Old);
2687	RTGCPHYS GCPhysCR3;
2688	switch (pVCpu->pgm.s.enmGuestMode)
2689	{
2690	case PGMMODE_PAE:
2691	case PGMMODE_PAE_NX:
2692	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
2693	break;
2694	case PGMMODE_AMD64:
2695	case PGMMODE_AMD64_NX:
2696	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
2697	break;
2698	default:
2699	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
2700	break;
2701	}
2702	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2703
2704	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2705	{
2706	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2707	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2708	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2709	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2710	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2711	}
2712
2713	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2714	if ( rc == VINF_PGM_SYNC_CR3
2715	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2716	{
2717	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2718	#ifdef IN_RING3
2719	rc = pgmPoolSyncCR3(pVCpu);
2720	#else
2721	if (rc == VINF_PGM_SYNC_CR3)
2722	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3Old;
2723	return VINF_PGM_SYNC_CR3;
2724	#endif
2725	}
2726	AssertRCReturn(rc, rc);
2727	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2728	}
2729
2730	/*
2731	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2732	*/
2733	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
2734
2735	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2736	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2737	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2738	rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2739
2740	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
2741	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
2742	if (rc == VINF_SUCCESS)
2743	{
2744	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2745	{
2746	/* Go back to ring 3 if a pgm pool sync is again pending. */
2747	return VINF_PGM_SYNC_CR3;
2748	}
2749
2750	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2751	{
2752	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2753	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2754	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2755	}
2756
2757	/*
2758	* Check if we have a pending update of the CR3 monitoring.
2759	*/
2760	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
2761	{
2762	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2763	Assert(!pVCpu->CTX_SUFF(pVM)->pgm.s.fMappingsFixed);
2764	Assert(pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
2765	}
2766	}
2767
2768	/*
2769	* Now flush the CR3 (guest context).
2770	*/
2771	if (rc == VINF_SUCCESS)
2772	PGM_INVL_VCPU_TLBS(pVCpu);
2773	return rc;
2774	}
2775
2776
2777	/**
2778	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
2779	*
2780	* @returns VBox status code, with the following informational code for
2781	* VM scheduling.
2782	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
2783	* @retval VINF_PGM_CHANGE_MODE if we're in RC the mode changes. This will
2784	* NOT be returned in ring-3 or ring-0.
2785	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
2786	*
2787	* @param pVCpu The cross context virtual CPU structure.
2788	* @param cr0 The new cr0.
2789	* @param cr4 The new cr4.
2790	* @param efer The new extended feature enable register.
2791	*/
2792	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer)
2793	{
2794	VMCPU_ASSERT_EMT(pVCpu);
2795
2796	/*
2797	* Calc the new guest mode.
2798	*
2799	* Note! We check PG before PE and without requiring PE because of the
2800	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
2801	*/
2802	PGMMODE enmGuestMode;
2803	if (cr0 & X86_CR0_PG)
2804	{
2805	if (!(cr4 & X86_CR4_PAE))
2806	{
2807	bool const fPse = !!(cr4 & X86_CR4_PSE);
2808	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
2809	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
2810	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
2811	enmGuestMode = PGMMODE_32_BIT;
2812	}
2813	else if (!(efer & MSR_K6_EFER_LME))
2814	{
2815	if (!(efer & MSR_K6_EFER_NXE))
2816	enmGuestMode = PGMMODE_PAE;
2817	else
2818	enmGuestMode = PGMMODE_PAE_NX;
2819	}
2820	else
2821	{
2822	if (!(efer & MSR_K6_EFER_NXE))
2823	enmGuestMode = PGMMODE_AMD64;
2824	else
2825	enmGuestMode = PGMMODE_AMD64_NX;
2826	}
2827	}
2828	else if (!(cr0 & X86_CR0_PE))
2829	enmGuestMode = PGMMODE_REAL;
2830	else
2831	enmGuestMode = PGMMODE_PROTECTED;
2832
2833	/*
2834	* Did it change?
2835	*/
2836	if (pVCpu->pgm.s.enmGuestMode == enmGuestMode)
2837	return VINF_SUCCESS;
2838
2839	/* Flush the TLB */
2840	PGM_INVL_VCPU_TLBS(pVCpu);
2841	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode);
2842	}
2843
2844
2845	/**
2846	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
2847	*
2848	* @returns PGM_TYPE_*.
2849	* @param pgmMode The mode value to convert.
2850	*/
2851	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
2852	{
2853	switch (pgmMode)
2854	{
2855	case PGMMODE_REAL: return PGM_TYPE_REAL;
2856	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
2857	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
2858	case PGMMODE_PAE:
2859	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
2860	case PGMMODE_AMD64:
2861	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
2862	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
2863	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
2864	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
2865	case PGMMODE_EPT: return PGM_TYPE_EPT;
2866	case PGMMODE_NONE: return PGM_TYPE_NONE;
2867	default:
2868	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
2869	}
2870	}
2871
2872
2873	/**
2874	* Calculates the shadow paging mode.
2875	*
2876	* @returns The shadow paging mode.
2877	* @param pVM The cross context VM structure.
2878	* @param enmGuestMode The guest mode.
2879	* @param enmHostMode The host mode.
2880	* @param enmShadowMode The current shadow mode.
2881	*/
2882	static PGMMODE pgmCalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
2883	{
2884	switch (enmGuestMode)
2885	{
2886	/*
2887	* When switching to real or protected mode we don't change
2888	* anything since it's likely that we'll switch back pretty soon.
2889	*
2890	* During pgmR3InitPaging we'll end up here with PGMMODE_INVALID
2891	* and is supposed to determine which shadow paging and switcher to
2892	* use during init.
2893	*/
2894	case PGMMODE_REAL:
2895	case PGMMODE_PROTECTED:
2896	if ( enmShadowMode != PGMMODE_INVALID
2897	&& VM_IS_RAW_MODE_ENABLED(pVM) /* always switch in hm and nem modes! */)
2898	break; /* (no change) */
2899
2900	switch (enmHostMode)
2901	{
2902	case SUPPAGINGMODE_32_BIT:
2903	case SUPPAGINGMODE_32_BIT_GLOBAL:
2904	enmShadowMode = PGMMODE_32_BIT;
2905	break;
2906
2907	case SUPPAGINGMODE_PAE:
2908	case SUPPAGINGMODE_PAE_NX:
2909	case SUPPAGINGMODE_PAE_GLOBAL:
2910	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2911	enmShadowMode = PGMMODE_PAE;
2912	break;
2913
2914	case SUPPAGINGMODE_AMD64:
2915	case SUPPAGINGMODE_AMD64_GLOBAL:
2916	case SUPPAGINGMODE_AMD64_NX:
2917	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2918	enmShadowMode = PGMMODE_PAE;
2919	break;
2920
2921	default:
2922	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2923	}
2924	break;
2925
2926	case PGMMODE_32_BIT:
2927	switch (enmHostMode)
2928	{
2929	case SUPPAGINGMODE_32_BIT:
2930	case SUPPAGINGMODE_32_BIT_GLOBAL:
2931	enmShadowMode = PGMMODE_32_BIT;
2932	break;
2933
2934	case SUPPAGINGMODE_PAE:
2935	case SUPPAGINGMODE_PAE_NX:
2936	case SUPPAGINGMODE_PAE_GLOBAL:
2937	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2938	enmShadowMode = PGMMODE_PAE;
2939	break;
2940
2941	case SUPPAGINGMODE_AMD64:
2942	case SUPPAGINGMODE_AMD64_GLOBAL:
2943	case SUPPAGINGMODE_AMD64_NX:
2944	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2945	enmShadowMode = PGMMODE_PAE;
2946	break;
2947
2948	default:
2949	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2950	}
2951	break;
2952
2953	case PGMMODE_PAE:
2954	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
2955	switch (enmHostMode)
2956	{
2957	case SUPPAGINGMODE_32_BIT:
2958	case SUPPAGINGMODE_32_BIT_GLOBAL:
2959	enmShadowMode = PGMMODE_PAE;
2960	break;
2961
2962	case SUPPAGINGMODE_PAE:
2963	case SUPPAGINGMODE_PAE_NX:
2964	case SUPPAGINGMODE_PAE_GLOBAL:
2965	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2966	enmShadowMode = PGMMODE_PAE;
2967	break;
2968
2969	case SUPPAGINGMODE_AMD64:
2970	case SUPPAGINGMODE_AMD64_GLOBAL:
2971	case SUPPAGINGMODE_AMD64_NX:
2972	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2973	enmShadowMode = PGMMODE_PAE;
2974	break;
2975
2976	default:
2977	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2978	}
2979	break;
2980
2981	case PGMMODE_AMD64:
2982	case PGMMODE_AMD64_NX:
2983	switch (enmHostMode)
2984	{
2985	case SUPPAGINGMODE_32_BIT:
2986	case SUPPAGINGMODE_32_BIT_GLOBAL:
2987	enmShadowMode = PGMMODE_AMD64;
2988	break;
2989
2990	case SUPPAGINGMODE_PAE:
2991	case SUPPAGINGMODE_PAE_NX:
2992	case SUPPAGINGMODE_PAE_GLOBAL:
2993	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2994	enmShadowMode = PGMMODE_AMD64;
2995	break;
2996
2997	case SUPPAGINGMODE_AMD64:
2998	case SUPPAGINGMODE_AMD64_GLOBAL:
2999	case SUPPAGINGMODE_AMD64_NX:
3000	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3001	enmShadowMode = PGMMODE_AMD64;
3002	break;
3003
3004	default:
3005	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3006	}
3007	break;
3008
3009	default:
3010	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3011	}
3012
3013	/*
3014	* Override the shadow mode when NEM or nested paging is active.
3015	*/
3016	if (VM_IS_NEM_ENABLED(pVM))
3017	{
3018	pVM->pgm.s.fNestedPaging = true;
3019	enmShadowMode = PGMMODE_NONE;
3020	}
3021	else
3022	{
3023	bool fNestedPaging = HMIsNestedPagingActive(pVM);
3024	pVM->pgm.s.fNestedPaging = fNestedPaging;
3025	if (fNestedPaging)
3026	{
3027	if (HMIsVmxActive(pVM))
3028	enmShadowMode = PGMMODE_EPT;
3029	else
3030	{
3031	/* The nested SVM paging depends on the host one. */
3032	Assert(HMIsSvmActive(pVM));
3033	if ( enmGuestMode == PGMMODE_AMD64
3034	\|\| enmGuestMode == PGMMODE_AMD64_NX)
3035	enmShadowMode = PGMMODE_NESTED_AMD64;
3036	else
3037	switch (pVM->pgm.s.enmHostMode)
3038	{
3039	case SUPPAGINGMODE_32_BIT:
3040	case SUPPAGINGMODE_32_BIT_GLOBAL:
3041	enmShadowMode = PGMMODE_NESTED_32BIT;
3042	break;
3043
3044	case SUPPAGINGMODE_PAE:
3045	case SUPPAGINGMODE_PAE_GLOBAL:
3046	case SUPPAGINGMODE_PAE_NX:
3047	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3048	enmShadowMode = PGMMODE_NESTED_PAE;
3049	break;
3050
3051	case SUPPAGINGMODE_AMD64:
3052	case SUPPAGINGMODE_AMD64_GLOBAL:
3053	case SUPPAGINGMODE_AMD64_NX:
3054	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3055	enmShadowMode = PGMMODE_NESTED_AMD64;
3056	break;
3057
3058	default:
3059	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3060	}
3061	}
3062	}
3063	}
3064
3065	return enmShadowMode;
3066	}
3067
3068
3069	/**
3070	* Performs the actual mode change.
3071	* This is called by PGMChangeMode and pgmR3InitPaging().
3072	*
3073	* @returns VBox status code. May suspend or power off the VM on error, but this
3074	* will trigger using FFs and not informational status codes.
3075	*
3076	* @param pVM The cross context VM structure.
3077	* @param pVCpu The cross context virtual CPU structure.
3078	* @param enmGuestMode The new guest mode. This is assumed to be different from
3079	* the current mode.
3080	*/
3081	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode)
3082	{
3083	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3084	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3085
3086	/*
3087	* Calc the shadow mode and switcher.
3088	*/
3089	PGMMODE enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3090
3091	/*
3092	* Exit old mode(s).
3093	*/
3094	/* shadow */
3095	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3096	{
3097	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3098	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3099	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3100	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3101	{
3102	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3103	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3104	}
3105	}
3106	else
3107	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3108
3109	/* guest */
3110	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3111	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3112	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3113	{
3114	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3115	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3116	}
3117	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3118
3119	/*
3120	* Change the paging mode data indexes.
3121	*/
3122	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3123	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3124	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3125	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3126	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3127	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPDE, VERR_PGM_MODE_IPE);
3128	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3129	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3130	#ifdef IN_RING3
3131	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3132	#endif
3133
3134	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3135	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3136	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3137	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3138	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3139	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3140	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3141	#ifdef IN_RING3
3142	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3143	#endif
3144
3145	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3146	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3147	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3148	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3149	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3150	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3151	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
3152	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3153	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3154	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3155	#ifdef VBOX_STRICT
3156	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3157	#endif
3158
3159	/*
3160	* Enter new shadow mode (if changed).
3161	*/
3162	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3163	{
3164	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3165	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu, enmGuestMode >= PGMMODE_AMD64);
3166	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3167	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3168	}
3169
3170	/*
3171	* Always flag the necessary updates
3172	*/
3173	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3174
3175	/*
3176	* Enter the new guest and shadow+guest modes.
3177	*/
3178	/* Calc the new CR3 value. */
3179	RTGCPHYS GCPhysCR3;
3180	switch (enmGuestMode)
3181	{
3182	case PGMMODE_REAL:
3183	case PGMMODE_PROTECTED:
3184	GCPhysCR3 = NIL_RTGCPHYS;
3185	break;
3186
3187	case PGMMODE_32_BIT:
3188	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3189	break;
3190
3191	case PGMMODE_PAE_NX:
3192	case PGMMODE_PAE:
3193	if (!pVM->cpum.ro.GuestFeatures.fPae)
3194	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3195	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3196	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3197	break;
3198
3199	#ifdef VBOX_WITH_64_BITS_GUESTS
3200	case PGMMODE_AMD64_NX:
3201	case PGMMODE_AMD64:
3202	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3203	break;
3204	#endif
3205	default:
3206	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3207	}
3208
3209	/* Enter the new guest mode. */
3210	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3211	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3212	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3213
3214	/* Set the new guest CR3. */
3215	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3216
3217	/* status codes. */
3218	AssertRC(rc);
3219	AssertRC(rc2);
3220	if (RT_SUCCESS(rc))
3221	{
3222	rc = rc2;
3223	if (RT_SUCCESS(rc)) /* no informational status codes. */
3224	rc = VINF_SUCCESS;
3225	}
3226
3227	/*
3228	* Notify HM.
3229	*/
3230	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3231	return rc;
3232	}
3233
3234
3235	/**
3236	* Called by CPUM or REM when CR0.WP changes to 1.
3237	*
3238	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3239	* @thread EMT
3240	*/
3241	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3242	{
3243	/*
3244	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3245	*
3246	* Use the counter to judge whether there might be pool pages with active
3247	* hacks in them. If there are, we will be running the risk of messing up
3248	* the guest by allowing it to write to read-only pages. Thus, we have to
3249	* clear the page pool ASAP if there is the slightest chance.
3250	*/
3251	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3252	{
3253	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3254
3255	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3256	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3257	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3258	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3259	}
3260	}
3261
3262
3263	/**
3264	* Gets the current guest paging mode.
3265	*
3266	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3267	*
3268	* @returns The current paging mode.
3269	* @param pVCpu The cross context virtual CPU structure.
3270	*/
3271	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3272	{
3273	return pVCpu->pgm.s.enmGuestMode;
3274	}
3275
3276
3277	/**
3278	* Gets the current shadow paging mode.
3279	*
3280	* @returns The current paging mode.
3281	* @param pVCpu The cross context virtual CPU structure.
3282	*/
3283	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3284	{
3285	return pVCpu->pgm.s.enmShadowMode;
3286	}
3287
3288
3289	/**
3290	* Gets the current host paging mode.
3291	*
3292	* @returns The current paging mode.
3293	* @param pVM The cross context VM structure.
3294	*/
3295	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3296	{
3297	switch (pVM->pgm.s.enmHostMode)
3298	{
3299	case SUPPAGINGMODE_32_BIT:
3300	case SUPPAGINGMODE_32_BIT_GLOBAL:
3301	return PGMMODE_32_BIT;
3302
3303	case SUPPAGINGMODE_PAE:
3304	case SUPPAGINGMODE_PAE_GLOBAL:
3305	return PGMMODE_PAE;
3306
3307	case SUPPAGINGMODE_PAE_NX:
3308	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3309	return PGMMODE_PAE_NX;
3310
3311	case SUPPAGINGMODE_AMD64:
3312	case SUPPAGINGMODE_AMD64_GLOBAL:
3313	return PGMMODE_AMD64;
3314
3315	case SUPPAGINGMODE_AMD64_NX:
3316	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3317	return PGMMODE_AMD64_NX;
3318
3319	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3320	}
3321
3322	return PGMMODE_INVALID;
3323	}
3324
3325
3326	/**
3327	* Get mode name.
3328	*
3329	* @returns read-only name string.
3330	* @param enmMode The mode which name is desired.
3331	*/
3332	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3333	{
3334	switch (enmMode)
3335	{
3336	case PGMMODE_REAL: return "Real";
3337	case PGMMODE_PROTECTED: return "Protected";
3338	case PGMMODE_32_BIT: return "32-bit";
3339	case PGMMODE_PAE: return "PAE";
3340	case PGMMODE_PAE_NX: return "PAE+NX";
3341	case PGMMODE_AMD64: return "AMD64";
3342	case PGMMODE_AMD64_NX: return "AMD64+NX";
3343	case PGMMODE_NESTED_32BIT: return "Nested-32";
3344	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3345	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3346	case PGMMODE_EPT: return "EPT";
3347	case PGMMODE_NONE: return "None";
3348	default: return "unknown mode value";
3349	}
3350	}
3351
3352
3353	/**
3354	* Gets the physical address represented in the guest CR3 as PGM sees it.
3355	*
3356	* This is mainly for logging and debugging.
3357	*
3358	* @returns PGM's guest CR3 value.
3359	* @param pVCpu The cross context virtual CPU structure.
3360	*/
3361	VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
3362	{
3363	return pVCpu->pgm.s.GCPhysCR3;
3364	}
3365
3366
3367
3368	/**
3369	* Notification from CPUM that the EFER.NXE bit has changed.
3370	*
3371	* @param pVCpu The cross context virtual CPU structure of the CPU for
3372	* which EFER changed.
3373	* @param fNxe The new NXE state.
3374	*/
3375	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3376	{
3377	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3378	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3379
3380	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3381	if (fNxe)
3382	{
3383	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3384	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3385	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3386	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3387	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3388	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3389	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3390	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3391	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3392	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3393	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3394
3395	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3396	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3397	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3398	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3399	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3400	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3401	}
3402	else
3403	{
3404	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3405	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3406	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3407	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3408	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3409	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3410	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3411	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3412	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3413	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3414	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3415
3416	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3417	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3418	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3419	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3420	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3421	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3422	}
3423	}
3424
3425
3426	/**
3427	* Check if any pgm pool pages are marked dirty (not monitored)
3428	*
3429	* @returns bool locked/not locked
3430	* @param pVM The cross context VM structure.
3431	*/
3432	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3433	{
3434	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3435	}
3436
3437
3438	/**
3439	* Check if this VCPU currently owns the PGM lock.
3440	*
3441	* @returns bool owner/not owner
3442	* @param pVM The cross context VM structure.
3443	*/
3444	VMMDECL(bool) PGMIsLockOwner(PVM pVM)
3445	{
3446	return PDMCritSectIsOwner(&pVM->pgm.s.CritSectX);
3447	}
3448
3449
3450	/**
3451	* Enable or disable large page usage
3452	*
3453	* @returns VBox status code.
3454	* @param pVM The cross context VM structure.
3455	* @param fUseLargePages Use/not use large pages
3456	*/
3457	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3458	{
3459	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3460
3461	pVM->fUseLargePages = fUseLargePages;
3462	return VINF_SUCCESS;
3463	}
3464
3465
3466	/**
3467	* Acquire the PGM lock.
3468	*
3469	* @returns VBox status code
3470	* @param pVM The cross context VM structure.
3471	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3472	*/
3473	#if (defined(VBOX_STRICT) && defined(IN_RING3)) \|\| defined(DOXYGEN_RUNNING)
3474	int pgmLockDebug(PVMCC pVM, RT_SRC_POS_DECL)
3475	#else
3476	int pgmLock(PVMCC pVM)
3477	#endif
3478	{
3479	#if defined(VBOX_STRICT) && defined(IN_RING3)
3480	int rc = PDMCritSectEnterDebug(&pVM->pgm.s.CritSectX, VERR_SEM_BUSY, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3481	#else
3482	int rc = PDMCritSectEnter(&pVM->pgm.s.CritSectX, VERR_SEM_BUSY);
3483	#endif
3484	#ifdef IN_RING0
3485	if (rc == VERR_SEM_BUSY)
3486	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_LOCK, 0);
3487	#endif
3488	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
3489	return rc;
3490	}
3491
3492
3493	/**
3494	* Release the PGM lock.
3495	*
3496	* @returns VBox status code
3497	* @param pVM The cross context VM structure.
3498	*/
3499	void pgmUnlock(PVM pVM)
3500	{
3501	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3502	pVM->pgm.s.cDeprecatedPageLocks = 0;
3503	int rc = PDMCritSectLeave(&pVM->pgm.s.CritSectX);
3504	if (rc == VINF_SEM_NESTED)
3505	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3506	}
3507
3508	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3509
3510	/**
3511	* Common worker for pgmRZDynMapGCPageOffInlined and pgmRZDynMapGCPageV2Inlined.
3512	*
3513	* @returns VBox status code.
3514	* @param pVM The cross context VM structure.
3515	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3516	* @param GCPhys The guest physical address of the page to map. The
3517	* offset bits are not ignored.
3518	* @param ppv Where to return the address corresponding to @a GCPhys.
3519	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3520	*/
3521	int pgmRZDynMapGCPageCommon(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
3522	{
3523	pgmLock(pVM);
3524
3525	/*
3526	* Convert it to a writable page and it on to the dynamic mapper.
3527	*/
3528	int rc;
3529	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
3530	if (RT_LIKELY(pPage))
3531	{
3532	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3533	if (RT_SUCCESS(rc))
3534	{
3535	void *pv;
3536	rc = pgmRZDynMapHCPageInlined(pVCpu, PGM_PAGE_GET_HCPHYS(pPage), &pv RTLOG_COMMA_SRC_POS_ARGS);
3537	if (RT_SUCCESS(rc))
3538	ppv = (void )((uintptr_t)pv \| ((uintptr_t)GCPhys & PAGE_OFFSET_MASK));
3539	}
3540	else
3541	AssertRC(rc);
3542	}
3543	else
3544	{
3545	AssertMsgFailed(("Invalid physical address %RGp!\n", GCPhys));
3546	rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3547	}
3548
3549	pgmUnlock(pVM);
3550	return rc;
3551	}
3552
3553	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
3554	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3555
3556	/** Format handler for PGMPAGE.
3557	* @copydoc FNRTSTRFORMATTYPE */
3558	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3559	const char pszType, void const pvValue,
3560	int cchWidth, int cchPrecision, unsigned fFlags,
3561	void *pvUser)
3562	{
3563	size_t cch;
3564	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
3565	if (RT_VALID_PTR(pPage))
3566	{
3567	char szTmp[64+80];
3568
3569	cch = 0;
3570
3571	/* The single char state stuff. */
3572	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
3573	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
3574
3575	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
3576	if (IS_PART_INCLUDED(5))
3577	{
3578	static const char s_achHandlerStates[4] = { '-', 't', 'w', 'a' };
3579	szTmp[cch++] = s_achHandlerStates[PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)];
3580	}
3581
3582	/* The type. */
3583	if (IS_PART_INCLUDED(4))
3584	{
3585	szTmp[cch++] = ':';
3586	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
3587	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
3588	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
3589	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
3590	}
3591
3592	/* The numbers. */
3593	if (IS_PART_INCLUDED(3))
3594	{
3595	szTmp[cch++] = ':';
3596	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3597	}
3598
3599	if (IS_PART_INCLUDED(2))
3600	{
3601	szTmp[cch++] = ':';
3602	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
3603	}
3604
3605	if (IS_PART_INCLUDED(6))
3606	{
3607	szTmp[cch++] = ':';
3608	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
3609	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
3610	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
3611	}
3612	# undef IS_PART_INCLUDED
3613
3614	cch = pfnOutput(pvArgOutput, szTmp, cch);
3615	}
3616	else
3617	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
3618	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
3619	return cch;
3620	}
3621
3622
3623	/** Format handler for PGMRAMRANGE.
3624	* @copydoc FNRTSTRFORMATTYPE */
3625	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3626	const char pszType, void const pvValue,
3627	int cchWidth, int cchPrecision, unsigned fFlags,
3628	void *pvUser)
3629	{
3630	size_t cch;
3631	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
3632	if (VALID_PTR(pRam))
3633	{
3634	char szTmp[80];
3635	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
3636	cch = pfnOutput(pvArgOutput, szTmp, cch);
3637	}
3638	else
3639	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
3640	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
3641	return cch;
3642	}
3643
3644	/** Format type andlers to be registered/deregistered. */
3645	static const struct
3646	{
3647	char szType[24];
3648	PFNRTSTRFORMATTYPE pfnHandler;
3649	} g_aPgmFormatTypes[] =
3650	{
3651	{ "pgmpage", pgmFormatTypeHandlerPage },
3652	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
3653	};
3654
3655	#endif /* !IN_R0 \|\| LOG_ENABLED */
3656
3657	/**
3658	* Registers the global string format types.
3659	*
3660	* This should be called at module load time or in some other manner that ensure
3661	* that it's called exactly one time.
3662	*
3663	* @returns IPRT status code on RTStrFormatTypeRegister failure.
3664	*/
3665	VMMDECL(int) PGMRegisterStringFormatTypes(void)
3666	{
3667	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3668	int rc = VINF_SUCCESS;
3669	unsigned i;
3670	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3671	{
3672	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3673	# ifdef IN_RING0
3674	if (rc == VERR_ALREADY_EXISTS)
3675	{
3676	/* in case of cleanup failure in ring-0 */
3677	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3678	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3679	}
3680	# endif
3681	}
3682	if (RT_FAILURE(rc))
3683	while (i-- > 0)
3684	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3685
3686	return rc;
3687	#else
3688	return VINF_SUCCESS;
3689	#endif
3690	}
3691
3692
3693	/**
3694	* Deregisters the global string format types.
3695	*
3696	* This should be called at module unload time or in some other manner that
3697	* ensure that it's called exactly one time.
3698	*/
3699	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
3700	{
3701	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3702	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3703	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3704	#endif
3705	}
3706
3707	#ifdef VBOX_STRICT
3708
3709	# ifndef PGM_WITHOUT_MAPPINGS
3710	/**
3711	* Asserts that there are no mapping conflicts.
3712	*
3713	* @returns Number of conflicts.
3714	* @param pVM The cross context VM structure.
3715	*/
3716	VMMDECL(unsigned) PGMAssertNoMappingConflicts(PVM pVM)
3717	{
3718	unsigned cErrors = 0;
3719
3720	/* Only applies to raw mode -> 1 VPCU */
3721	Assert(pVM->cCpus == 1);
3722	PVMCPU pVCpu = &VMCC_GET_CPU_0(pVM);
3723
3724	/*
3725	* Check for mapping conflicts.
3726	*/
3727	for (PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
3728	pMapping;
3729	pMapping = pMapping->CTX_SUFF(pNext))
3730	{
3731	/** @todo This is slow and should be optimized, but since it's just assertions I don't care now. */
3732	for (RTGCPTR GCPtr = pMapping->GCPtr; GCPtr <= pMapping->GCPtrLast; GCPtr += PAGE_SIZE)
3733	{
3734	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)GCPtr, NULL, NULL);
3735	if (rc != VERR_PAGE_TABLE_NOT_PRESENT)
3736	{
3737	AssertMsgFailed(("Conflict at %RGv with %s\n", GCPtr, R3STRING(pMapping->pszDesc)));
3738	cErrors++;
3739	break;
3740	}
3741	}
3742	}
3743
3744	return cErrors;
3745	}
3746	# endif /* !PGM_WITHOUT_MAPPINGS */
3747
3748
3749	/**
3750	* Asserts that everything related to the guest CR3 is correctly shadowed.
3751	*
3752	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
3753	* and assert the correctness of the guest CR3 mapping before asserting that the
3754	* shadow page tables is in sync with the guest page tables.
3755	*
3756	* @returns Number of conflicts.
3757	* @param pVM The cross context VM structure.
3758	* @param pVCpu The cross context virtual CPU structure.
3759	* @param cr3 The current guest CR3 register value.
3760	* @param cr4 The current guest CR4 register value.
3761	*/
3762	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
3763	{
3764	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
3765
3766	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
3767	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
3768	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
3769
3770	pgmLock(pVM);
3771	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
3772	pgmUnlock(pVM);
3773
3774	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
3775	return cErrors;
3776	}
3777
3778	#endif /* VBOX_STRICT */
3779

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

© 2023 Oracle
Contact – Privacy policy – Terms of Use