VMXAllTemplate.cpp.h@ 100594

Last change on this file since 100594 was 100244, checked in by vboxsync, 18 months ago
VMM: Nested VMX: bugref:10318 Guru meditate if the nested-EPT trap handler does not indicate an EPT misconfig. when we expect it to.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 526.4 KB

Line
1	/* $Id: VMXAllTemplate.cpp.h 100244 2023-06-22 03:59:01Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Code template for our own hypervisor and the NEM darwin backend using Apple's Hypervisor.framework.
4	*/
5
6	/*
7	* Copyright (C) 2012-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Defined Constants And Macros *
31	*********************************************************************************************************************************/
32	#if !defined(VMX_VMCS_WRITE_16) \|\| !defined(VMX_VMCS_WRITE_32) \|\| !defined(VMX_VMCS_WRITE_64) \|\| !defined(VMX_VMCS_WRITE_64)
33	# error "At least one of the VMX_VMCS_WRITE_16, VMX_VMCS_WRITE_32, VMX_VMCS_WRITE_64 or VMX_VMCS_WRITE_64 is missing"
34	#endif
35
36
37	#if !defined(VMX_VMCS_READ_16) \|\| !defined(VMX_VMCS_READ_32) \|\| !defined(VMX_VMCS_READ_64) \|\| !defined(VMX_VMCS_READ_64)
38	# error "At least one of the VMX_VMCS_READ_16, VMX_VMCS_READ_32, VMX_VMCS_READ_64 or VMX_VMCS_READ_64 is missing"
39	#endif
40
41	/** Enables condensing of VMREAD instructions, see vmxHCReadToTransient(). */
42	#define HMVMX_WITH_CONDENSED_VMREADS
43
44	/** Use the function table. */
45	#define HMVMX_USE_FUNCTION_TABLE
46
47	/** Determine which tagged-TLB flush handler to use. */
48	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
49	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
50	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
51	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
52
53	/** Assert that all the given fields have been read from the VMCS. */
54	#ifdef VBOX_STRICT
55	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
56	do { \
57	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
58	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
59	} while (0)
60	#else
61	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
62	#endif
63
64	/**
65	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
66	* guest using hardware-assisted VMX.
67	*
68	* This excludes state like GPRs (other than RSP) which are always are
69	* swapped and restored across the world-switch and also registers like EFER,
70	* MSR which cannot be modified by the guest without causing a VM-exit.
71	*/
72	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
73	\| CPUMCTX_EXTRN_RFLAGS \
74	\| CPUMCTX_EXTRN_RSP \
75	\| CPUMCTX_EXTRN_SREG_MASK \
76	\| CPUMCTX_EXTRN_TABLE_MASK \
77	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
78	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
79	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
80	\| CPUMCTX_EXTRN_TSC_AUX \
81	\| CPUMCTX_EXTRN_OTHER_MSRS \
82	\| CPUMCTX_EXTRN_CR0 \
83	\| CPUMCTX_EXTRN_CR3 \
84	\| CPUMCTX_EXTRN_CR4 \
85	\| CPUMCTX_EXTRN_DR7 \
86	\| CPUMCTX_EXTRN_HWVIRT \
87	\| CPUMCTX_EXTRN_INHIBIT_INT \
88	\| CPUMCTX_EXTRN_INHIBIT_NMI)
89
90	/**
91	* Exception bitmap mask for real-mode guests (real-on-v86).
92	*
93	* We need to intercept all exceptions manually except:
94	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
95	* due to bugs in Intel CPUs.
96	* - \#PF need not be intercepted even in real-mode if we have nested paging
97	* support.
98	*/
99	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
100	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
101	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
102	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
103	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
104	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
105	\| RT_BIT(X86_XCPT_XF))
106
107	/** Maximum VM-instruction error number. */
108	#define HMVMX_INSTR_ERROR_MAX 28
109
110	/** Profiling macro. */
111	#ifdef HM_PROFILE_EXIT_DISPATCH
112	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
113	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
114	#else
115	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
116	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
117	#endif
118
119	#ifndef IN_NEM_DARWIN
120	/** Assert that preemption is disabled or covered by thread-context hooks. */
121	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
122	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
123
124	/** Assert that we haven't migrated CPUs when thread-context hooks are not
125	* used. */
126	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
127	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
128	("Illegal migration! Entered on CPU %u Current %u\n", \
129	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()))
130	#else
131	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) do { } while (0)
132	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) do { } while (0)
133	#endif
134
135	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
136	* context. */
137	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Log the VM-exit reason with an easily visible marker to identify it in a
142	* potential sea of logging data. */
143	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
144	do { \
145	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
146	HMGetVmxExitName(a_uExitReason))); \
147	} while (0) \
148
149
150	/*********************************************************************************************************************************
151	* Structures and Typedefs *
152	*********************************************************************************************************************************/
153	/**
154	* Memory operand read or write access.
155	*/
156	typedef enum VMXMEMACCESS
157	{
158	VMXMEMACCESS_READ = 0,
159	VMXMEMACCESS_WRITE = 1
160	} VMXMEMACCESS;
161
162
163	/**
164	* VMX VM-exit handler.
165	*
166	* @returns Strict VBox status code (i.e. informational status codes too).
167	* @param pVCpu The cross context virtual CPU structure.
168	* @param pVmxTransient The VMX-transient structure.
169	*/
170	#ifndef HMVMX_USE_FUNCTION_TABLE
171	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
172	#else
173	typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
174	/** Pointer to VM-exit handler. */
175	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
176	#endif
177
178	/**
179	* VMX VM-exit handler, non-strict status code.
180	*
181	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
182	*
183	* @returns VBox status code, no informational status code returned.
184	* @param pVCpu The cross context virtual CPU structure.
185	* @param pVmxTransient The VMX-transient structure.
186	*
187	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
188	* use of that status code will be replaced with VINF_EM_SOMETHING
189	* later when switching over to IEM.
190	*/
191	#ifndef HMVMX_USE_FUNCTION_TABLE
192	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
193	#else
194	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
195	#endif
196
197
198	/*********************************************************************************************************************************
199	* Internal Functions *
200	*********************************************************************************************************************************/
201	#ifndef HMVMX_USE_FUNCTION_TABLE
202	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
203	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
204	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
205	#else
206	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
207	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
208	#endif
209	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
210	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
211	#endif
212
213	static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
214
215	/** @name VM-exit handler prototypes.
216	* @{
217	*/
218	static FNVMXEXITHANDLER vmxHCExitXcptOrNmi;
219	static FNVMXEXITHANDLER vmxHCExitExtInt;
220	static FNVMXEXITHANDLER vmxHCExitTripleFault;
221	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindow;
222	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindow;
223	static FNVMXEXITHANDLER vmxHCExitTaskSwitch;
224	static FNVMXEXITHANDLER vmxHCExitCpuid;
225	static FNVMXEXITHANDLER vmxHCExitGetsec;
226	static FNVMXEXITHANDLER vmxHCExitHlt;
227	static FNVMXEXITHANDLERNSRC vmxHCExitInvd;
228	static FNVMXEXITHANDLER vmxHCExitInvlpg;
229	static FNVMXEXITHANDLER vmxHCExitRdpmc;
230	static FNVMXEXITHANDLER vmxHCExitVmcall;
231	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
232	static FNVMXEXITHANDLER vmxHCExitVmclear;
233	static FNVMXEXITHANDLER vmxHCExitVmlaunch;
234	static FNVMXEXITHANDLER vmxHCExitVmptrld;
235	static FNVMXEXITHANDLER vmxHCExitVmptrst;
236	static FNVMXEXITHANDLER vmxHCExitVmread;
237	static FNVMXEXITHANDLER vmxHCExitVmresume;
238	static FNVMXEXITHANDLER vmxHCExitVmwrite;
239	static FNVMXEXITHANDLER vmxHCExitVmxoff;
240	static FNVMXEXITHANDLER vmxHCExitVmxon;
241	static FNVMXEXITHANDLER vmxHCExitInvvpid;
242	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
243	static FNVMXEXITHANDLER vmxHCExitInvept;
244	# endif
245	#endif
246	static FNVMXEXITHANDLER vmxHCExitRdtsc;
247	static FNVMXEXITHANDLER vmxHCExitMovCRx;
248	static FNVMXEXITHANDLER vmxHCExitMovDRx;
249	static FNVMXEXITHANDLER vmxHCExitIoInstr;
250	static FNVMXEXITHANDLER vmxHCExitRdmsr;
251	static FNVMXEXITHANDLER vmxHCExitWrmsr;
252	static FNVMXEXITHANDLER vmxHCExitMwait;
253	static FNVMXEXITHANDLER vmxHCExitMtf;
254	static FNVMXEXITHANDLER vmxHCExitMonitor;
255	static FNVMXEXITHANDLER vmxHCExitPause;
256	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThreshold;
257	static FNVMXEXITHANDLER vmxHCExitApicAccess;
258	static FNVMXEXITHANDLER vmxHCExitEptViolation;
259	static FNVMXEXITHANDLER vmxHCExitEptMisconfig;
260	static FNVMXEXITHANDLER vmxHCExitRdtscp;
261	static FNVMXEXITHANDLER vmxHCExitPreemptTimer;
262	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvd;
263	static FNVMXEXITHANDLER vmxHCExitXsetbv;
264	static FNVMXEXITHANDLER vmxHCExitInvpcid;
265	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
266	static FNVMXEXITHANDLERNSRC vmxHCExitSetPendingXcptUD;
267	#endif
268	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestState;
269	static FNVMXEXITHANDLERNSRC vmxHCExitErrUnexpected;
270	/** @} */
271
272	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
273	/** @name Nested-guest VM-exit handler prototypes.
274	* @{
275	*/
276	static FNVMXEXITHANDLER vmxHCExitXcptOrNmiNested;
277	static FNVMXEXITHANDLER vmxHCExitTripleFaultNested;
278	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindowNested;
279	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindowNested;
280	static FNVMXEXITHANDLER vmxHCExitTaskSwitchNested;
281	static FNVMXEXITHANDLER vmxHCExitHltNested;
282	static FNVMXEXITHANDLER vmxHCExitInvlpgNested;
283	static FNVMXEXITHANDLER vmxHCExitRdpmcNested;
284	static FNVMXEXITHANDLER vmxHCExitVmreadVmwriteNested;
285	static FNVMXEXITHANDLER vmxHCExitRdtscNested;
286	static FNVMXEXITHANDLER vmxHCExitMovCRxNested;
287	static FNVMXEXITHANDLER vmxHCExitMovDRxNested;
288	static FNVMXEXITHANDLER vmxHCExitIoInstrNested;
289	static FNVMXEXITHANDLER vmxHCExitRdmsrNested;
290	static FNVMXEXITHANDLER vmxHCExitWrmsrNested;
291	static FNVMXEXITHANDLER vmxHCExitMwaitNested;
292	static FNVMXEXITHANDLER vmxHCExitMtfNested;
293	static FNVMXEXITHANDLER vmxHCExitMonitorNested;
294	static FNVMXEXITHANDLER vmxHCExitPauseNested;
295	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThresholdNested;
296	static FNVMXEXITHANDLER vmxHCExitApicAccessNested;
297	static FNVMXEXITHANDLER vmxHCExitApicWriteNested;
298	static FNVMXEXITHANDLER vmxHCExitVirtEoiNested;
299	static FNVMXEXITHANDLER vmxHCExitRdtscpNested;
300	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvdNested;
301	static FNVMXEXITHANDLER vmxHCExitInvpcidNested;
302	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestStateNested;
303	static FNVMXEXITHANDLER vmxHCExitInstrNested;
304	static FNVMXEXITHANDLER vmxHCExitInstrWithInfoNested;
305	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
306	static FNVMXEXITHANDLER vmxHCExitEptViolationNested;
307	static FNVMXEXITHANDLER vmxHCExitEptMisconfigNested;
308	# endif
309	/** @} */
310	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
311
312
313	/*********************************************************************************************************************************
314	* Global Variables *
315	*********************************************************************************************************************************/
316	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
317	/**
318	* Array of all VMCS fields.
319	* Any fields added to the VT-x spec. should be added here.
320	*
321	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
322	* of nested-guests.
323	*/
324	static const uint32_t g_aVmcsFields[] =
325	{
326	/* 16-bit control fields. */
327	VMX_VMCS16_VPID,
328	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
329	VMX_VMCS16_EPTP_INDEX,
330	VMX_VMCS16_HLAT_PREFIX_SIZE,
331
332	/* 16-bit guest-state fields. */
333	VMX_VMCS16_GUEST_ES_SEL,
334	VMX_VMCS16_GUEST_CS_SEL,
335	VMX_VMCS16_GUEST_SS_SEL,
336	VMX_VMCS16_GUEST_DS_SEL,
337	VMX_VMCS16_GUEST_FS_SEL,
338	VMX_VMCS16_GUEST_GS_SEL,
339	VMX_VMCS16_GUEST_LDTR_SEL,
340	VMX_VMCS16_GUEST_TR_SEL,
341	VMX_VMCS16_GUEST_INTR_STATUS,
342	VMX_VMCS16_GUEST_PML_INDEX,
343
344	/* 16-bits host-state fields. */
345	VMX_VMCS16_HOST_ES_SEL,
346	VMX_VMCS16_HOST_CS_SEL,
347	VMX_VMCS16_HOST_SS_SEL,
348	VMX_VMCS16_HOST_DS_SEL,
349	VMX_VMCS16_HOST_FS_SEL,
350	VMX_VMCS16_HOST_GS_SEL,
351	VMX_VMCS16_HOST_TR_SEL,
352
353	/* 64-bit control fields. */
354	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
355	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
356	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
357	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
358	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
359	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
360	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
361	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
362	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
363	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
364	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
365	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
366	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
367	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
368	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
369	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
370	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
371	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
372	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
373	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
374	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
375	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
376	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
377	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
378	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
379	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
380	VMX_VMCS64_CTRL_EPTP_FULL,
381	VMX_VMCS64_CTRL_EPTP_HIGH,
382	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
383	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
384	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
385	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
386	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
387	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
388	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
389	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
390	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
391	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
392	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
393	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
394	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
395	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
396	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL,
397	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH,
398	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
399	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
400	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
401	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
402	VMX_VMCS64_CTRL_SPPTP_FULL,
403	VMX_VMCS64_CTRL_SPPTP_HIGH,
404	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
405	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
406	VMX_VMCS64_CTRL_PROC_EXEC3_FULL,
407	VMX_VMCS64_CTRL_PROC_EXEC3_HIGH,
408	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL,
409	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH,
410	VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_FULL,
411	VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_HIGH,
412	VMX_VMCS64_CTRL_HLAT_PTR_FULL,
413	VMX_VMCS64_CTRL_HLAT_PTR_HIGH,
414	VMX_VMCS64_CTRL_EXIT2_FULL,
415	VMX_VMCS64_CTRL_EXIT2_HIGH,
416
417	/* 64-bit read-only data fields. */
418	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
419	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
420
421	/* 64-bit guest-state fields. */
422	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
423	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
424	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
425	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
426	VMX_VMCS64_GUEST_PAT_FULL,
427	VMX_VMCS64_GUEST_PAT_HIGH,
428	VMX_VMCS64_GUEST_EFER_FULL,
429	VMX_VMCS64_GUEST_EFER_HIGH,
430	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
431	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
432	VMX_VMCS64_GUEST_PDPTE0_FULL,
433	VMX_VMCS64_GUEST_PDPTE0_HIGH,
434	VMX_VMCS64_GUEST_PDPTE1_FULL,
435	VMX_VMCS64_GUEST_PDPTE1_HIGH,
436	VMX_VMCS64_GUEST_PDPTE2_FULL,
437	VMX_VMCS64_GUEST_PDPTE2_HIGH,
438	VMX_VMCS64_GUEST_PDPTE3_FULL,
439	VMX_VMCS64_GUEST_PDPTE3_HIGH,
440	VMX_VMCS64_GUEST_BNDCFGS_FULL,
441	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
442	VMX_VMCS64_GUEST_RTIT_CTL_FULL,
443	VMX_VMCS64_GUEST_RTIT_CTL_HIGH,
444	VMX_VMCS64_GUEST_PKRS_FULL,
445	VMX_VMCS64_GUEST_PKRS_HIGH,
446
447	/* 64-bit host-state fields. */
448	VMX_VMCS64_HOST_PAT_FULL,
449	VMX_VMCS64_HOST_PAT_HIGH,
450	VMX_VMCS64_HOST_EFER_FULL,
451	VMX_VMCS64_HOST_EFER_HIGH,
452	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
453	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
454	VMX_VMCS64_HOST_PKRS_FULL,
455	VMX_VMCS64_HOST_PKRS_HIGH,
456
457	/* 32-bit control fields. */
458	VMX_VMCS32_CTRL_PIN_EXEC,
459	VMX_VMCS32_CTRL_PROC_EXEC,
460	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
461	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
462	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
463	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
464	VMX_VMCS32_CTRL_EXIT,
465	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
466	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
467	VMX_VMCS32_CTRL_ENTRY,
468	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
469	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
470	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
471	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
472	VMX_VMCS32_CTRL_TPR_THRESHOLD,
473	VMX_VMCS32_CTRL_PROC_EXEC2,
474	VMX_VMCS32_CTRL_PLE_GAP,
475	VMX_VMCS32_CTRL_PLE_WINDOW,
476
477	/* 32-bits read-only fields. */
478	VMX_VMCS32_RO_VM_INSTR_ERROR,
479	VMX_VMCS32_RO_EXIT_REASON,
480	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
481	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
482	VMX_VMCS32_RO_IDT_VECTORING_INFO,
483	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
484	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
485	VMX_VMCS32_RO_EXIT_INSTR_INFO,
486
487	/* 32-bit guest-state fields. */
488	VMX_VMCS32_GUEST_ES_LIMIT,
489	VMX_VMCS32_GUEST_CS_LIMIT,
490	VMX_VMCS32_GUEST_SS_LIMIT,
491	VMX_VMCS32_GUEST_DS_LIMIT,
492	VMX_VMCS32_GUEST_FS_LIMIT,
493	VMX_VMCS32_GUEST_GS_LIMIT,
494	VMX_VMCS32_GUEST_LDTR_LIMIT,
495	VMX_VMCS32_GUEST_TR_LIMIT,
496	VMX_VMCS32_GUEST_GDTR_LIMIT,
497	VMX_VMCS32_GUEST_IDTR_LIMIT,
498	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
499	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
500	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
501	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
502	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
503	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
504	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
505	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
506	VMX_VMCS32_GUEST_INT_STATE,
507	VMX_VMCS32_GUEST_ACTIVITY_STATE,
508	VMX_VMCS32_GUEST_SMBASE,
509	VMX_VMCS32_GUEST_SYSENTER_CS,
510	VMX_VMCS32_PREEMPT_TIMER_VALUE,
511
512	/* 32-bit host-state fields. */
513	VMX_VMCS32_HOST_SYSENTER_CS,
514
515	/* Natural-width control fields. */
516	VMX_VMCS_CTRL_CR0_MASK,
517	VMX_VMCS_CTRL_CR4_MASK,
518	VMX_VMCS_CTRL_CR0_READ_SHADOW,
519	VMX_VMCS_CTRL_CR4_READ_SHADOW,
520	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
521	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
522	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
523	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
524
525	/* Natural-width read-only data fields. */
526	VMX_VMCS_RO_EXIT_QUALIFICATION,
527	VMX_VMCS_RO_IO_RCX,
528	VMX_VMCS_RO_IO_RSI,
529	VMX_VMCS_RO_IO_RDI,
530	VMX_VMCS_RO_IO_RIP,
531	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
532
533	/* Natural-width guest-state field */
534	VMX_VMCS_GUEST_CR0,
535	VMX_VMCS_GUEST_CR3,
536	VMX_VMCS_GUEST_CR4,
537	VMX_VMCS_GUEST_ES_BASE,
538	VMX_VMCS_GUEST_CS_BASE,
539	VMX_VMCS_GUEST_SS_BASE,
540	VMX_VMCS_GUEST_DS_BASE,
541	VMX_VMCS_GUEST_FS_BASE,
542	VMX_VMCS_GUEST_GS_BASE,
543	VMX_VMCS_GUEST_LDTR_BASE,
544	VMX_VMCS_GUEST_TR_BASE,
545	VMX_VMCS_GUEST_GDTR_BASE,
546	VMX_VMCS_GUEST_IDTR_BASE,
547	VMX_VMCS_GUEST_DR7,
548	VMX_VMCS_GUEST_RSP,
549	VMX_VMCS_GUEST_RIP,
550	VMX_VMCS_GUEST_RFLAGS,
551	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
552	VMX_VMCS_GUEST_SYSENTER_ESP,
553	VMX_VMCS_GUEST_SYSENTER_EIP,
554	VMX_VMCS_GUEST_S_CET,
555	VMX_VMCS_GUEST_SSP,
556	VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR,
557
558	/* Natural-width host-state fields */
559	VMX_VMCS_HOST_CR0,
560	VMX_VMCS_HOST_CR3,
561	VMX_VMCS_HOST_CR4,
562	VMX_VMCS_HOST_FS_BASE,
563	VMX_VMCS_HOST_GS_BASE,
564	VMX_VMCS_HOST_TR_BASE,
565	VMX_VMCS_HOST_GDTR_BASE,
566	VMX_VMCS_HOST_IDTR_BASE,
567	VMX_VMCS_HOST_SYSENTER_ESP,
568	VMX_VMCS_HOST_SYSENTER_EIP,
569	VMX_VMCS_HOST_RSP,
570	VMX_VMCS_HOST_RIP,
571	VMX_VMCS_HOST_S_CET,
572	VMX_VMCS_HOST_SSP,
573	VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR
574	};
575	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
576
577	#ifdef HMVMX_USE_FUNCTION_TABLE
578	/**
579	* VMX_EXIT dispatch table.
580	*/
581	static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
582	{
583	/* 0 VMX_EXIT_XCPT_OR_NMI */ { vmxHCExitXcptOrNmi },
584	/* 1 VMX_EXIT_EXT_INT */ { vmxHCExitExtInt },
585	/* 2 VMX_EXIT_TRIPLE_FAULT */ { vmxHCExitTripleFault },
586	/* 3 VMX_EXIT_INIT_SIGNAL */ { vmxHCExitErrUnexpected },
587	/* 4 VMX_EXIT_SIPI */ { vmxHCExitErrUnexpected },
588	/* 5 VMX_EXIT_IO_SMI */ { vmxHCExitErrUnexpected },
589	/* 6 VMX_EXIT_SMI */ { vmxHCExitErrUnexpected },
590	/* 7 VMX_EXIT_INT_WINDOW */ { vmxHCExitIntWindow },
591	/* 8 VMX_EXIT_NMI_WINDOW */ { vmxHCExitNmiWindow },
592	/* 9 VMX_EXIT_TASK_SWITCH */ { vmxHCExitTaskSwitch },
593	/* 10 VMX_EXIT_CPUID */ { vmxHCExitCpuid },
594	/* 11 VMX_EXIT_GETSEC */ { vmxHCExitGetsec },
595	/* 12 VMX_EXIT_HLT */ { vmxHCExitHlt },
596	/* 13 VMX_EXIT_INVD */ { vmxHCExitInvd },
597	/* 14 VMX_EXIT_INVLPG */ { vmxHCExitInvlpg },
598	/* 15 VMX_EXIT_RDPMC */ { vmxHCExitRdpmc },
599	/* 16 VMX_EXIT_RDTSC */ { vmxHCExitRdtsc },
600	/* 17 VMX_EXIT_RSM */ { vmxHCExitErrUnexpected },
601	/* 18 VMX_EXIT_VMCALL */ { vmxHCExitVmcall },
602	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
603	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitVmclear },
604	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitVmlaunch },
605	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitVmptrld },
606	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitVmptrst },
607	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitVmread },
608	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitVmresume },
609	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitVmwrite },
610	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitVmxoff },
611	/* 27 VMX_EXIT_VMXON */ { vmxHCExitVmxon },
612	#else
613	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitSetPendingXcptUD },
614	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitSetPendingXcptUD },
615	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitSetPendingXcptUD },
616	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitSetPendingXcptUD },
617	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitSetPendingXcptUD },
618	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitSetPendingXcptUD },
619	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitSetPendingXcptUD },
620	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitSetPendingXcptUD },
621	/* 27 VMX_EXIT_VMXON */ { vmxHCExitSetPendingXcptUD },
622	#endif
623	/* 28 VMX_EXIT_MOV_CRX */ { vmxHCExitMovCRx },
624	/* 29 VMX_EXIT_MOV_DRX */ { vmxHCExitMovDRx },
625	/* 30 VMX_EXIT_IO_INSTR */ { vmxHCExitIoInstr },
626	/* 31 VMX_EXIT_RDMSR */ { vmxHCExitRdmsr },
627	/* 32 VMX_EXIT_WRMSR */ { vmxHCExitWrmsr },
628	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ { vmxHCExitErrInvalidGuestState },
629	/* 34 VMX_EXIT_ERR_MSR_LOAD */ { vmxHCExitErrUnexpected },
630	/* 35 UNDEFINED */ { vmxHCExitErrUnexpected },
631	/* 36 VMX_EXIT_MWAIT */ { vmxHCExitMwait },
632	/* 37 VMX_EXIT_MTF */ { vmxHCExitMtf },
633	/* 38 UNDEFINED */ { vmxHCExitErrUnexpected },
634	/* 39 VMX_EXIT_MONITOR */ { vmxHCExitMonitor },
635	/* 40 VMX_EXIT_PAUSE */ { vmxHCExitPause },
636	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ { vmxHCExitErrUnexpected },
637	/* 42 UNDEFINED */ { vmxHCExitErrUnexpected },
638	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ { vmxHCExitTprBelowThreshold },
639	/* 44 VMX_EXIT_APIC_ACCESS */ { vmxHCExitApicAccess },
640	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ { vmxHCExitErrUnexpected },
641	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ { vmxHCExitErrUnexpected },
642	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ { vmxHCExitErrUnexpected },
643	/* 48 VMX_EXIT_EPT_VIOLATION */ { vmxHCExitEptViolation },
644	/* 49 VMX_EXIT_EPT_MISCONFIG */ { vmxHCExitEptMisconfig },
645	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
646	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitInvept },
647	#else
648	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitSetPendingXcptUD },
649	#endif
650	/* 51 VMX_EXIT_RDTSCP */ { vmxHCExitRdtscp },
651	/* 52 VMX_EXIT_PREEMPT_TIMER */ { vmxHCExitPreemptTimer },
652	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
653	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitInvvpid },
654	#else
655	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitSetPendingXcptUD },
656	#endif
657	/* 54 VMX_EXIT_WBINVD */ { vmxHCExitWbinvd },
658	/* 55 VMX_EXIT_XSETBV */ { vmxHCExitXsetbv },
659	/* 56 VMX_EXIT_APIC_WRITE */ { vmxHCExitErrUnexpected },
660	/* 57 VMX_EXIT_RDRAND */ { vmxHCExitErrUnexpected },
661	/* 58 VMX_EXIT_INVPCID */ { vmxHCExitInvpcid },
662	/* 59 VMX_EXIT_VMFUNC */ { vmxHCExitErrUnexpected },
663	/* 60 VMX_EXIT_ENCLS */ { vmxHCExitErrUnexpected },
664	/* 61 VMX_EXIT_RDSEED */ { vmxHCExitErrUnexpected },
665	/* 62 VMX_EXIT_PML_FULL */ { vmxHCExitErrUnexpected },
666	/* 63 VMX_EXIT_XSAVES */ { vmxHCExitErrUnexpected },
667	/* 64 VMX_EXIT_XRSTORS */ { vmxHCExitErrUnexpected },
668	/* 65 UNDEFINED */ { vmxHCExitErrUnexpected },
669	/* 66 VMX_EXIT_SPP_EVENT */ { vmxHCExitErrUnexpected },
670	/* 67 VMX_EXIT_UMWAIT */ { vmxHCExitErrUnexpected },
671	/* 68 VMX_EXIT_TPAUSE */ { vmxHCExitErrUnexpected },
672	/* 69 VMX_EXIT_LOADIWKEY */ { vmxHCExitErrUnexpected },
673	};
674	#endif /* HMVMX_USE_FUNCTION_TABLE */
675
676	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
677	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
678	{
679	/* 0 */ "(Not Used)",
680	/* 1 */ "VMCALL executed in VMX root operation.",
681	/* 2 */ "VMCLEAR with invalid physical address.",
682	/* 3 */ "VMCLEAR with VMXON pointer.",
683	/* 4 */ "VMLAUNCH with non-clear VMCS.",
684	/* 5 */ "VMRESUME with non-launched VMCS.",
685	/* 6 */ "VMRESUME after VMXOFF",
686	/* 7 */ "VM-entry with invalid control fields.",
687	/* 8 */ "VM-entry with invalid host state fields.",
688	/* 9 */ "VMPTRLD with invalid physical address.",
689	/* 10 */ "VMPTRLD with VMXON pointer.",
690	/* 11 */ "VMPTRLD with incorrect revision identifier.",
691	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
692	/* 13 */ "VMWRITE to read-only VMCS component.",
693	/* 14 */ "(Not Used)",
694	/* 15 */ "VMXON executed in VMX root operation.",
695	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
696	/* 17 */ "VM-entry with non-launched executing VMCS.",
697	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
698	/* 19 */ "VMCALL with non-clear VMCS.",
699	/* 20 */ "VMCALL with invalid VM-exit control fields.",
700	/* 21 */ "(Not Used)",
701	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
702	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
703	/* 24 */ "VMCALL with invalid SMM-monitor features.",
704	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
705	/* 26 */ "VM-entry with events blocked by MOV SS.",
706	/* 27 */ "(Not Used)",
707	/* 28 */ "Invalid operand to INVEPT/INVVPID."
708	};
709	#endif /* VBOX_STRICT && LOG_ENABLED */
710
711
712	/**
713	* Gets the CR0 guest/host mask.
714	*
715	* These bits typically does not change through the lifetime of a VM. Any bit set in
716	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
717	* by the guest.
718	*
719	* @returns The CR0 guest/host mask.
720	* @param pVCpu The cross context virtual CPU structure.
721	*/
722	static uint64_t vmxHCGetFixedCr0Mask(PCVMCPUCC pVCpu)
723	{
724	/*
725	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
726	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
727	*
728	* Furthermore, modifications to any bits that are reserved/unspecified currently
729	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
730	* when future CPUs specify and use currently reserved/unspecified bits.
731	*/
732	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
733	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
734	* and @bugref{6944}. */
735	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
736	AssertCompile(RT_HI_U32(VMX_EXIT_HOST_CR0_IGNORE_MASK) == UINT32_C(0xffffffff)); /* Paranoia. */
737	return ( X86_CR0_PE
738	\| X86_CR0_NE
739	\| (VM_IS_VMX_NESTED_PAGING(pVM) ? 0 : X86_CR0_WP)
740	\| X86_CR0_PG
741	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
742	}
743
744
745	/**
746	* Gets the CR4 guest/host mask.
747	*
748	* These bits typically does not change through the lifetime of a VM. Any bit set in
749	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
750	* by the guest.
751	*
752	* @returns The CR4 guest/host mask.
753	* @param pVCpu The cross context virtual CPU structure.
754	*/
755	static uint64_t vmxHCGetFixedCr4Mask(PCVMCPUCC pVCpu)
756	{
757	/*
758	* We construct a mask of all CR4 bits that the guest can modify without causing
759	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
760	* a VM-exit when the guest attempts to modify them when executing using
761	* hardware-assisted VMX.
762	*
763	* When a feature is not exposed to the guest (and may be present on the host),
764	* we want to intercept guest modifications to the bit so we can emulate proper
765	* behavior (e.g., #GP).
766	*
767	* Furthermore, only modifications to those bits that don't require immediate
768	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
769	* depends on CR3 which might not always be the guest value while executing
770	* using hardware-assisted VMX.
771	*/
772	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
773	bool fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
774	#ifdef IN_NEM_DARWIN
775	bool fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
776	#endif
777	bool fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
778
779	/*
780	* Paranoia.
781	* Ensure features exposed to the guest are present on the host.
782	*/
783	AssertStmt(!fFsGsBase \|\| g_CpumHostFeatures.s.fFsGsBase, fFsGsBase = 0);
784	#ifdef IN_NEM_DARWIN
785	AssertStmt(!fXSaveRstor \|\| g_CpumHostFeatures.s.fXSaveRstor, fXSaveRstor = 0);
786	#endif
787	AssertStmt(!fFxSaveRstor \|\| g_CpumHostFeatures.s.fFxSaveRstor, fFxSaveRstor = 0);
788
789	uint64_t const fGstMask = X86_CR4_PVI
790	\| X86_CR4_TSD
791	\| X86_CR4_DE
792	\| X86_CR4_MCE
793	\| X86_CR4_PCE
794	\| X86_CR4_OSXMMEEXCPT
795	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
796	#ifdef IN_NEM_DARWIN /* On native VT-x setting OSXSAVE must exit as we need to load guest XCR0 (see
797	fLoadSaveGuestXcr0). These exits are not needed on Darwin as that's not our problem. */
798	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
799	#endif
800	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0);
801	return ~fGstMask;
802	}
803
804
805	/**
806	* Adds one or more exceptions to the exception bitmap and commits it to the current
807	* VMCS.
808	*
809	* @param pVCpu The cross context virtual CPU structure.
810	* @param pVmxTransient The VMX-transient structure.
811	* @param uXcptMask The exception(s) to add.
812	*/
813	static void vmxHCAddXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
814	{
815	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
816	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
817	if ((uXcptBitmap & uXcptMask) != uXcptMask)
818	{
819	uXcptBitmap \|= uXcptMask;
820	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
821	AssertRC(rc);
822	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
823	}
824	}
825
826
827	/**
828	* Adds an exception to the exception bitmap and commits it to the current VMCS.
829	*
830	* @param pVCpu The cross context virtual CPU structure.
831	* @param pVmxTransient The VMX-transient structure.
832	* @param uXcpt The exception to add.
833	*/
834	static void vmxHCAddXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
835	{
836	Assert(uXcpt <= X86_XCPT_LAST);
837	vmxHCAddXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT_32(uXcpt));
838	}
839
840
841	/**
842	* Remove one or more exceptions from the exception bitmap and commits it to the
843	* current VMCS.
844	*
845	* This takes care of not removing the exception intercept if a nested-guest
846	* requires the exception to be intercepted.
847	*
848	* @returns VBox status code.
849	* @param pVCpu The cross context virtual CPU structure.
850	* @param pVmxTransient The VMX-transient structure.
851	* @param uXcptMask The exception(s) to remove.
852	*/
853	static int vmxHCRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
854	{
855	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
856	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
857	if (uXcptBitmap & uXcptMask)
858	{
859	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
860	if (!pVmxTransient->fIsNestedGuest)
861	{ /* likely */ }
862	else
863	uXcptMask &= ~pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap;
864	#endif
865	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
866	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
867	\| RT_BIT(X86_XCPT_DE)
868	\| RT_BIT(X86_XCPT_NM)
869	\| RT_BIT(X86_XCPT_TS)
870	\| RT_BIT(X86_XCPT_UD)
871	\| RT_BIT(X86_XCPT_NP)
872	\| RT_BIT(X86_XCPT_SS)
873	\| RT_BIT(X86_XCPT_GP)
874	\| RT_BIT(X86_XCPT_PF)
875	\| RT_BIT(X86_XCPT_MF));
876	#elif defined(HMVMX_ALWAYS_TRAP_PF)
877	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
878	#endif
879	if (uXcptMask)
880	{
881	/* Validate we are not removing any essential exception intercepts. */
882	#ifndef IN_NEM_DARWIN
883	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
884	#else
885	Assert(!(uXcptMask & RT_BIT(X86_XCPT_PF)));
886	#endif
887	NOREF(pVCpu);
888	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
889	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
890
891	/* Remove it from the exception bitmap. */
892	uXcptBitmap &= ~uXcptMask;
893
894	/* Commit and update the cache if necessary. */
895	if (pVmcsInfo->u32XcptBitmap != uXcptBitmap)
896	{
897	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
898	AssertRC(rc);
899	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
900	}
901	}
902	}
903	return VINF_SUCCESS;
904	}
905
906
907	/**
908	* Remove an exceptions from the exception bitmap and commits it to the current
909	* VMCS.
910	*
911	* @returns VBox status code.
912	* @param pVCpu The cross context virtual CPU structure.
913	* @param pVmxTransient The VMX-transient structure.
914	* @param uXcpt The exception to remove.
915	*/
916	static int vmxHCRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
917	{
918	return vmxHCRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
919	}
920
921	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
922
923	/**
924	* Loads the shadow VMCS specified by the VMCS info. object.
925	*
926	* @returns VBox status code.
927	* @param pVmcsInfo The VMCS info. object.
928	*
929	* @remarks Can be called with interrupts disabled.
930	*/
931	static int vmxHCLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
932	{
933	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
934	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
935
936	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
937	if (RT_SUCCESS(rc))
938	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
939	return rc;
940	}
941
942
943	/**
944	* Clears the shadow VMCS specified by the VMCS info. object.
945	*
946	* @returns VBox status code.
947	* @param pVmcsInfo The VMCS info. object.
948	*
949	* @remarks Can be called with interrupts disabled.
950	*/
951	static int vmxHCClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
952	{
953	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
954	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
955
956	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
957	if (RT_SUCCESS(rc))
958	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
959	return rc;
960	}
961
962
963	/**
964	* Switches from and to the specified VMCSes.
965	*
966	* @returns VBox status code.
967	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
968	* @param pVmcsInfoTo The VMCS info. object we are switching to.
969	*
970	* @remarks Called with interrupts disabled.
971	*/
972	static int vmxHCSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
973	{
974	/*
975	* Clear the VMCS we are switching out if it has not already been cleared.
976	* This will sync any CPU internal data back to the VMCS.
977	*/
978	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
979	{
980	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
981	if (RT_SUCCESS(rc))
982	{
983	/*
984	* The shadow VMCS, if any, would not be active at this point since we
985	* would have cleared it while importing the virtual hardware-virtualization
986	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
987	* clear the shadow VMCS here, just assert for safety.
988	*/
989	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
990	}
991	else
992	return rc;
993	}
994
995	/*
996	* Clear the VMCS we are switching to if it has not already been cleared.
997	* This will initialize the VMCS launch state to "clear" required for loading it.
998	*
999	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1000	*/
1001	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1002	{
1003	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1004	if (RT_SUCCESS(rc))
1005	{ /* likely */ }
1006	else
1007	return rc;
1008	}
1009
1010	/*
1011	* Finally, load the VMCS we are switching to.
1012	*/
1013	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1014	}
1015
1016
1017	/**
1018	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1019	* caller.
1020	*
1021	* @returns VBox status code.
1022	* @param pVCpu The cross context virtual CPU structure.
1023	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1024	* true) or guest VMCS (pass false).
1025	*/
1026	static int vmxHCSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1027	{
1028	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1029	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1030
1031	PVMXVMCSINFO pVmcsInfoFrom;
1032	PVMXVMCSINFO pVmcsInfoTo;
1033	if (fSwitchToNstGstVmcs)
1034	{
1035	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfo;
1036	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1037	}
1038	else
1039	{
1040	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1041	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfo;
1042	}
1043
1044	/*
1045	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1046	* preemption hook code path acquires the current VMCS.
1047	*/
1048	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1049
1050	int rc = vmxHCSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1051	if (RT_SUCCESS(rc))
1052	{
1053	pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1054	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3 = fSwitchToNstGstVmcs;
1055
1056	/*
1057	* If we are switching to a VMCS that was executed on a different host CPU or was
1058	* never executed before, flag that we need to export the host state before executing
1059	* guest/nested-guest code using hardware-assisted VMX.
1060	*
1061	* This could probably be done in a preemptible context since the preemption hook
1062	* will flag the necessary change in host context. However, since preemption is
1063	* already disabled and to avoid making assumptions about host specific code in
1064	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1065	* disabled.
1066	*/
1067	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1068	{ /* likely */ }
1069	else
1070	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1071
1072	ASMSetFlags(fEFlags);
1073
1074	/*
1075	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1076	* flag that we need to update the host MSR values there. Even if we decide in the
1077	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1078	* if its content differs, we would have to update the host MSRs anyway.
1079	*/
1080	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
1081	}
1082	else
1083	ASMSetFlags(fEFlags);
1084	return rc;
1085	}
1086
1087	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1088	#ifdef VBOX_STRICT
1089
1090	/**
1091	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1092	* transient structure.
1093	*
1094	* @param pVCpu The cross context virtual CPU structure.
1095	* @param pVmxTransient The VMX-transient structure.
1096	*/
1097	DECLINLINE(void) vmxHCReadEntryIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1098	{
1099	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1100	AssertRC(rc);
1101	}
1102
1103
1104	/**
1105	* Reads the VM-entry exception error code field from the VMCS into
1106	* the VMX transient structure.
1107	*
1108	* @param pVCpu The cross context virtual CPU structure.
1109	* @param pVmxTransient The VMX-transient structure.
1110	*/
1111	DECLINLINE(void) vmxHCReadEntryXcptErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1112	{
1113	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1114	AssertRC(rc);
1115	}
1116
1117
1118	/**
1119	* Reads the VM-entry exception error code field from the VMCS into
1120	* the VMX transient structure.
1121	*
1122	* @param pVCpu The cross context virtual CPU structure.
1123	* @param pVmxTransient The VMX-transient structure.
1124	*/
1125	DECLINLINE(void) vmxHCReadEntryInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1126	{
1127	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1128	AssertRC(rc);
1129	}
1130
1131	#endif /* VBOX_STRICT */
1132
1133
1134	/**
1135	* Reads VMCS fields into the VMXTRANSIENT structure, slow path version.
1136	*
1137	* Don't call directly unless the it's likely that some or all of the fields
1138	* given in @a a_fReadMask have already been read.
1139	*
1140	* @tparam a_fReadMask The fields to read.
1141	* @param pVCpu The cross context virtual CPU structure.
1142	* @param pVmxTransient The VMX-transient structure.
1143	*/
1144	template<uint32_t const a_fReadMask>
1145	static void vmxHCReadToTransientSlow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1146	{
1147	AssertCompile((a_fReadMask & ~( HMVMX_READ_EXIT_QUALIFICATION
1148	\| HMVMX_READ_EXIT_INSTR_LEN
1149	\| HMVMX_READ_EXIT_INSTR_INFO
1150	\| HMVMX_READ_IDT_VECTORING_INFO
1151	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1152	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1153	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1154	\| HMVMX_READ_GUEST_LINEAR_ADDR
1155	\| HMVMX_READ_GUEST_PHYSICAL_ADDR
1156	\| HMVMX_READ_GUEST_PENDING_DBG_XCPTS
1157	)) == 0);
1158
1159	if ((pVmxTransient->fVmcsFieldsRead & a_fReadMask) != a_fReadMask)
1160	{
1161	uint32_t const fVmcsFieldsRead = pVmxTransient->fVmcsFieldsRead;
1162
1163	if ( (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION)
1164	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1165	{
1166	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1167	AssertRC(rc);
1168	}
1169	if ( (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN)
1170	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1171	{
1172	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1173	AssertRC(rc);
1174	}
1175	if ( (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO)
1176	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1177	{
1178	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1179	AssertRC(rc);
1180	}
1181	if ( (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO)
1182	&& !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1183	{
1184	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1185	AssertRC(rc);
1186	}
1187	if ( (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
1188	&& !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1189	{
1190	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1191	AssertRC(rc);
1192	}
1193	if ( (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO)
1194	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1195	{
1196	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1197	AssertRC(rc);
1198	}
1199	if ( (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
1200	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1201	{
1202	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1203	AssertRC(rc);
1204	}
1205	if ( (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR)
1206	&& !(fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1207	{
1208	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1209	AssertRC(rc);
1210	}
1211	if ( (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR)
1212	&& !(fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1213	{
1214	int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1215	AssertRC(rc);
1216	}
1217	if ( (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
1218	&& !(fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1219	{
1220	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1221	AssertRC(rc);
1222	}
1223
1224	pVmxTransient->fVmcsFieldsRead \|= a_fReadMask;
1225	}
1226	}
1227
1228
1229	/**
1230	* Reads VMCS fields into the VMXTRANSIENT structure.
1231	*
1232	* This optimizes for the case where none of @a a_fReadMask has been read yet,
1233	* generating an optimized read sequences w/o any conditionals between in
1234	* non-strict builds.
1235	*
1236	* @tparam a_fReadMask The fields to read. One or more of the
1237	* HMVMX_READ_XXX fields ORed together.
1238	* @param pVCpu The cross context virtual CPU structure.
1239	* @param pVmxTransient The VMX-transient structure.
1240	*/
1241	template<uint32_t const a_fReadMask>
1242	DECLINLINE(void) vmxHCReadToTransient(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1243	{
1244	AssertCompile((a_fReadMask & ~( HMVMX_READ_EXIT_QUALIFICATION
1245	\| HMVMX_READ_EXIT_INSTR_LEN
1246	\| HMVMX_READ_EXIT_INSTR_INFO
1247	\| HMVMX_READ_IDT_VECTORING_INFO
1248	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1249	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1250	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1251	\| HMVMX_READ_GUEST_LINEAR_ADDR
1252	\| HMVMX_READ_GUEST_PHYSICAL_ADDR
1253	\| HMVMX_READ_GUEST_PENDING_DBG_XCPTS
1254	)) == 0);
1255
1256	if (RT_LIKELY(!(pVmxTransient->fVmcsFieldsRead & a_fReadMask)))
1257	{
1258	if (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION)
1259	{
1260	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1261	AssertRC(rc);
1262	}
1263	if (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN)
1264	{
1265	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1266	AssertRC(rc);
1267	}
1268	if (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO)
1269	{
1270	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1271	AssertRC(rc);
1272	}
1273	if (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO)
1274	{
1275	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1276	AssertRC(rc);
1277	}
1278	if (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
1279	{
1280	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1281	AssertRC(rc);
1282	}
1283	if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO)
1284	{
1285	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1286	AssertRC(rc);
1287	}
1288	if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
1289	{
1290	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1291	AssertRC(rc);
1292	}
1293	if (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR)
1294	{
1295	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1296	AssertRC(rc);
1297	}
1298	if (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR)
1299	{
1300	int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1301	AssertRC(rc);
1302	}
1303	if (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
1304	{
1305	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1306	AssertRC(rc);
1307	}
1308
1309	pVmxTransient->fVmcsFieldsRead \|= a_fReadMask;
1310	}
1311	else
1312	{
1313	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatReadToTransientFallback);
1314	Log11Func(("a_fReadMask=%#x fVmcsFieldsRead=%#x => %#x - Taking inefficient code path!\n",
1315	a_fReadMask, pVmxTransient->fVmcsFieldsRead, a_fReadMask & pVmxTransient->fVmcsFieldsRead));
1316	vmxHCReadToTransientSlow<a_fReadMask>(pVCpu, pVmxTransient);
1317	}
1318	}
1319
1320
1321	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1322	/**
1323	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1324	*
1325	* @param pVCpu The cross context virtual CPU structure.
1326	* @param pVmxTransient The VMX-transient structure.
1327	*/
1328	static void vmxHCReadAllRoFieldsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1329	{
1330	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1331	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1332	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1333	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1334	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1335	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1336	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1337	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1338	rc \|= VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1339	AssertRC(rc);
1340	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1341	\| HMVMX_READ_EXIT_INSTR_LEN
1342	\| HMVMX_READ_EXIT_INSTR_INFO
1343	\| HMVMX_READ_IDT_VECTORING_INFO
1344	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1345	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1346	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1347	\| HMVMX_READ_GUEST_LINEAR_ADDR
1348	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1349	}
1350	#endif
1351
1352	/**
1353	* Verifies that our cached values of the VMCS fields are all consistent with
1354	* what's actually present in the VMCS.
1355	*
1356	* @returns VBox status code.
1357	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1358	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1359	* VMCS content. HMCPU error-field is
1360	* updated, see VMX_VCI_XXX.
1361	* @param pVCpu The cross context virtual CPU structure.
1362	* @param pVmcsInfo The VMCS info. object.
1363	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1364	*/
1365	static int vmxHCCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1366	{
1367	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
1368
1369	uint32_t u32Val;
1370	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
1371	AssertRC(rc);
1372	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
1373	("%s entry controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
1374	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_ENTRY,
1375	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1376
1377	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXIT, &u32Val);
1378	AssertRC(rc);
1379	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
1380	("%s exit controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
1381	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_EXIT,
1382	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1383
1384	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1385	AssertRC(rc);
1386	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
1387	("%s pin controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
1388	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1389	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1390
1391	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1392	AssertRC(rc);
1393	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
1394	("%s proc controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
1395	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1396	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1397
1398	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1399	{
1400	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1401	AssertRC(rc);
1402	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
1403	("%s proc2 controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
1404	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1405	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1406	}
1407
1408	uint64_t u64Val;
1409	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS)
1410	{
1411	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_PROC_EXEC3_FULL, &u64Val);
1412	AssertRC(rc);
1413	AssertMsgReturnStmt(pVmcsInfo->u64ProcCtls3 == u64Val,
1414	("%s proc3 controls mismatch: Cache=%#RX32 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64ProcCtls3, u64Val),
1415	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC3,
1416	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1417	}
1418
1419	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1420	AssertRC(rc);
1421	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
1422	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
1423	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1424	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1425
1426	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1427	AssertRC(rc);
1428	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
1429	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
1430	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1431	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1432
1433	NOREF(pcszVmcs);
1434	return VINF_SUCCESS;
1435	}
1436
1437
1438	/**
1439	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
1440	* VMCS.
1441	*
1442	* This is typically required when the guest changes paging mode.
1443	*
1444	* @returns VBox status code.
1445	* @param pVCpu The cross context virtual CPU structure.
1446	* @param pVmxTransient The VMX-transient structure.
1447	*
1448	* @remarks Requires EFER.
1449	* @remarks No-long-jump zone!!!
1450	*/
1451	static int vmxHCExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1452	{
1453	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
1454	{
1455	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1456	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1457
1458	/*
1459	* VM-entry controls.
1460	*/
1461	{
1462	uint32_t fVal = g_HmMsrs.u.vmx.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
1463	uint32_t const fZap = g_HmMsrs.u.vmx.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1464
1465	/*
1466	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
1467	* The first VT-x capable CPUs only supported the 1-setting of this bit.
1468	*
1469	* For nested-guests, this is a mandatory VM-entry control. It's also
1470	* required because we do not want to leak host bits to the nested-guest.
1471	*/
1472	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
1473
1474	/*
1475	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
1476	*
1477	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
1478	* required to get the nested-guest working with hardware-assisted VMX execution.
1479	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
1480	* can skip intercepting changes to the EFER MSR. This is why it needs to be done
1481	* here rather than while merging the guest VMCS controls.
1482	*/
1483	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
1484	{
1485	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
1486	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
1487	}
1488	else
1489	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
1490
1491	/*
1492	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
1493	*
1494	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
1495	* regardless of whether the nested-guest VMCS specifies it because we are free to
1496	* load whatever MSRs we require and we do not need to modify the guest visible copy
1497	* of the VM-entry MSR load area.
1498	*/
1499	if ( g_fHmVmxSupportsVmcsEfer
1500	#ifndef IN_NEM_DARWIN
1501	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient)
1502	#endif
1503	)
1504	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
1505	else
1506	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
1507
1508	/*
1509	* The following should -not- be set (since we're not in SMM mode):
1510	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
1511	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
1512	*/
1513
1514	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
1515	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
1516
1517	if ((fVal & fZap) == fVal)
1518	{ /* likely */ }
1519	else
1520	{
1521	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1522	g_HmMsrs.u.vmx.EntryCtls.n.allowed0, fVal, fZap));
1523	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_ENTRY;
1524	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1525	}
1526
1527	/* Commit it to the VMCS. */
1528	if (pVmcsInfo->u32EntryCtls != fVal)
1529	{
1530	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, fVal);
1531	AssertRC(rc);
1532	pVmcsInfo->u32EntryCtls = fVal;
1533	}
1534	}
1535
1536	/*
1537	* VM-exit controls.
1538	*/
1539	{
1540	uint32_t fVal = g_HmMsrs.u.vmx.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
1541	uint32_t const fZap = g_HmMsrs.u.vmx.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1542
1543	/*
1544	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
1545	* supported the 1-setting of this bit.
1546	*
1547	* For nested-guests, we set the "save debug controls" as the converse
1548	* "load debug controls" is mandatory for nested-guests anyway.
1549	*/
1550	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
1551
1552	/*
1553	* Set the host long mode active (EFER.LMA) bit (which Intel calls
1554	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
1555	* host EFER.LMA and EFER.LME bit to this value. See assertion in
1556	* vmxHCExportHostMsrs().
1557	*
1558	* For nested-guests, we always set this bit as we do not support 32-bit
1559	* hosts.
1560	*/
1561	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
1562
1563	#ifndef IN_NEM_DARWIN
1564	/*
1565	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
1566	*
1567	* For nested-guests, we should use the "save IA32_EFER" control if we also
1568	* used the "load IA32_EFER" control while exporting VM-entry controls.
1569	*/
1570	if ( g_fHmVmxSupportsVmcsEfer
1571	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
1572	{
1573	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
1574	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
1575	}
1576	#endif
1577
1578	/*
1579	* Enable saving of the VMX-preemption timer value on VM-exit.
1580	* For nested-guests, currently not exposed/used.
1581	*/
1582	/** @todo r=bird: Measure performance hit because of this vs. always rewriting
1583	* the timer value. */
1584	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
1585	{
1586	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
1587	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
1588	}
1589
1590	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
1591	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
1592
1593	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
1594	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
1595	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
1596
1597	if ((fVal & fZap) == fVal)
1598	{ /* likely */ }
1599	else
1600	{
1601	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1602	g_HmMsrs.u.vmx.ExitCtls.n.allowed0, fVal, fZap));
1603	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_EXIT;
1604	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1605	}
1606
1607	/* Commit it to the VMCS. */
1608	if (pVmcsInfo->u32ExitCtls != fVal)
1609	{
1610	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT, fVal);
1611	AssertRC(rc);
1612	pVmcsInfo->u32ExitCtls = fVal;
1613	}
1614	}
1615
1616	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
1617	}
1618	return VINF_SUCCESS;
1619	}
1620
1621
1622	/**
1623	* Sets the TPR threshold in the VMCS.
1624	*
1625	* @param pVCpu The cross context virtual CPU structure.
1626	* @param pVmcsInfo The VMCS info. object.
1627	* @param u32TprThreshold The TPR threshold (task-priority class only).
1628	*/
1629	DECLINLINE(void) vmxHCApicSetTprThreshold(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
1630	{
1631	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
1632	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
1633	RT_NOREF(pVmcsInfo);
1634	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
1635	AssertRC(rc);
1636	}
1637
1638
1639	/**
1640	* Exports the guest APIC TPR state into the VMCS.
1641	*
1642	* @param pVCpu The cross context virtual CPU structure.
1643	* @param pVmxTransient The VMX-transient structure.
1644	*
1645	* @remarks No-long-jump zone!!!
1646	*/
1647	static void vmxHCExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1648	{
1649	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
1650	{
1651	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
1652
1653	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1654	if (!pVmxTransient->fIsNestedGuest)
1655	{
1656	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1657	&& APICIsEnabled(pVCpu))
1658	{
1659	/*
1660	* Setup TPR shadowing.
1661	*/
1662	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
1663	{
1664	bool fPendingIntr = false;
1665	uint8_t u8Tpr = 0;
1666	uint8_t u8PendingIntr = 0;
1667	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
1668	AssertRC(rc);
1669
1670	/*
1671	* If there are interrupts pending but masked by the TPR, instruct VT-x to
1672	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
1673	* priority of the pending interrupt so we can deliver the interrupt. If there
1674	* are no interrupts pending, set threshold to 0 to not cause any
1675	* TPR-below-threshold VM-exits.
1676	*/
1677	uint32_t u32TprThreshold = 0;
1678	if (fPendingIntr)
1679	{
1680	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
1681	(which is the Task-Priority Class). */
1682	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
1683	const uint8_t u8TprPriority = u8Tpr >> 4;
1684	if (u8PendingPriority <= u8TprPriority)
1685	u32TprThreshold = u8PendingPriority;
1686	}
1687
1688	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
1689	}
1690	}
1691	}
1692	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
1693	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
1694	}
1695	}
1696
1697
1698	/**
1699	* Gets the guest interruptibility-state and updates related internal eflags
1700	* inhibition state.
1701	*
1702	* @returns Guest's interruptibility-state.
1703	* @param pVCpu The cross context virtual CPU structure.
1704	*
1705	* @remarks No-long-jump zone!!!
1706	*/
1707	static uint32_t vmxHCGetGuestIntrStateWithUpdate(PVMCPUCC pVCpu)
1708	{
1709	uint32_t fIntrState;
1710
1711	/*
1712	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
1713	*/
1714	if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))
1715	fIntrState = 0;
1716	else
1717	{
1718	/* If inhibition is active, RIP should've been imported from the VMCS already. */
1719	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
1720
1721	if (CPUMIsInInterruptShadowAfterSs(&pVCpu->cpum.GstCtx))
1722	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
1723	else
1724	{
1725	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
1726
1727	/* Block-by-STI must not be set when interrupts are disabled. */
1728	AssertStmt(pVCpu->cpum.GstCtx.eflags.Bits.u1IF, fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
1729	}
1730	}
1731
1732	/*
1733	* Check if we should inhibit NMI delivery.
1734	*/
1735	if (!CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
1736	{ /* likely */ }
1737	else
1738	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
1739
1740	/*
1741	* Validate.
1742	*/
1743	/* We don't support block-by-SMI yet.*/
1744	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
1745
1746	return fIntrState;
1747	}
1748
1749
1750	/**
1751	* Exports the exception intercepts required for guest execution in the VMCS.
1752	*
1753	* @param pVCpu The cross context virtual CPU structure.
1754	* @param pVmxTransient The VMX-transient structure.
1755	*
1756	* @remarks No-long-jump zone!!!
1757	*/
1758	static void vmxHCExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1759	{
1760	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
1761	{
1762	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
1763	if ( !pVmxTransient->fIsNestedGuest
1764	&& VCPU_2_VMXSTATE(pVCpu).fGIMTrapXcptUD)
1765	vmxHCAddXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
1766	else
1767	vmxHCRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
1768
1769	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
1770	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
1771	}
1772	}
1773
1774
1775	/**
1776	* Exports the guest's RIP into the guest-state area in the VMCS.
1777	*
1778	* @param pVCpu The cross context virtual CPU structure.
1779	*
1780	* @remarks No-long-jump zone!!!
1781	*/
1782	static void vmxHCExportGuestRip(PVMCPUCC pVCpu)
1783	{
1784	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RIP)
1785	{
1786	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
1787
1788	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
1789	AssertRC(rc);
1790
1791	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RIP);
1792	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
1793	}
1794	}
1795
1796
1797	/**
1798	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
1799	*
1800	* @param pVCpu The cross context virtual CPU structure.
1801	* @param pVmxTransient The VMX-transient structure.
1802	*
1803	* @remarks No-long-jump zone!!!
1804	*/
1805	static void vmxHCExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1806	{
1807	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
1808	{
1809	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
1810
1811	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits
1812	of RFLAGS are reserved (MBZ). We use bits 63:24 for internal purposes, so no need
1813	to assert this, the CPUMX86EFLAGS/CPUMX86RFLAGS union masks these off for us.
1814	Use 32-bit VMWRITE. */
1815	uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
1816	Assert((fEFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK);
1817	AssertMsg(!(fEFlags & ~(X86_EFL_LIVE_MASK \| X86_EFL_RA1_MASK)), ("%#x\n", fEFlags));
1818
1819	#ifndef IN_NEM_DARWIN
1820	/*
1821	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
1822	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
1823	* can run the real-mode guest code under Virtual 8086 mode.
1824	*/
1825	PVMXVMCSINFOSHARED pVmcsInfo = pVmxTransient->pVmcsInfo->pShared;
1826	if (pVmcsInfo->RealMode.fRealOnV86Active)
1827	{
1828	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
1829	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
1830	Assert(!pVmxTransient->fIsNestedGuest);
1831	pVmcsInfo->RealMode.Eflags.u32 = fEFlags; /* Save the original eflags of the real-mode guest. */
1832	fEFlags \|= X86_EFL_VM; /* Set the Virtual 8086 mode bit. */
1833	fEFlags &= ~X86_EFL_IOPL; /* Change IOPL to 0, otherwise certain instructions won't fault. */
1834	}
1835	#else
1836	RT_NOREF(pVmxTransient);
1837	#endif
1838
1839	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, fEFlags);
1840	AssertRC(rc);
1841
1842	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
1843	Log4Func(("eflags=%#RX32\n", fEFlags));
1844	}
1845	}
1846
1847
1848	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1849	/**
1850	* Copies the nested-guest VMCS to the shadow VMCS.
1851	*
1852	* @returns VBox status code.
1853	* @param pVCpu The cross context virtual CPU structure.
1854	* @param pVmcsInfo The VMCS info. object.
1855	*
1856	* @remarks No-long-jump zone!!!
1857	*/
1858	static int vmxHCCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1859	{
1860	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
1861	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
1862
1863	/*
1864	* Disable interrupts so we don't get preempted while the shadow VMCS is the
1865	* current VMCS, as we may try saving guest lazy MSRs.
1866	*
1867	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
1868	* calling the import VMCS code which is currently performing the guest MSR reads
1869	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
1870	* and the rest of the VMX leave session machinery.
1871	*/
1872	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1873
1874	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
1875	if (RT_SUCCESS(rc))
1876	{
1877	/*
1878	* Copy all guest read/write VMCS fields.
1879	*
1880	* We don't check for VMWRITE failures here for performance reasons and
1881	* because they are not expected to fail, barring irrecoverable conditions
1882	* like hardware errors.
1883	*/
1884	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
1885	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1886	{
1887	uint64_t u64Val;
1888	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
1889	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
1890	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
1891	}
1892
1893	/*
1894	* If the host CPU supports writing all VMCS fields, copy the guest read-only
1895	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
1896	*/
1897	if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
1898	{
1899	uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
1900	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
1901	{
1902	uint64_t u64Val;
1903	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsRoFields[i];
1904	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
1905	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
1906	}
1907	}
1908
1909	rc = vmxHCClearShadowVmcs(pVmcsInfo);
1910	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
1911	}
1912
1913	ASMSetFlags(fEFlags);
1914	return rc;
1915	}
1916
1917
1918	/**
1919	* Copies the shadow VMCS to the nested-guest VMCS.
1920	*
1921	* @returns VBox status code.
1922	* @param pVCpu The cross context virtual CPU structure.
1923	* @param pVmcsInfo The VMCS info. object.
1924	*
1925	* @remarks Called with interrupts disabled.
1926	*/
1927	static int vmxHCCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1928	{
1929	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1930	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
1931	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
1932
1933	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
1934	if (RT_SUCCESS(rc))
1935	{
1936	/*
1937	* Copy guest read/write fields from the shadow VMCS.
1938	* Guest read-only fields cannot be modified, so no need to copy them.
1939	*
1940	* We don't check for VMREAD failures here for performance reasons and
1941	* because they are not expected to fail, barring irrecoverable conditions
1942	* like hardware errors.
1943	*/
1944	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
1945	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1946	{
1947	uint64_t u64Val;
1948	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
1949	VMX_VMCS_READ_64(pVCpu, uVmcsField, &u64Val);
1950	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
1951	}
1952
1953	rc = vmxHCClearShadowVmcs(pVmcsInfo);
1954	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
1955	}
1956	return rc;
1957	}
1958
1959
1960	/**
1961	* Enables VMCS shadowing for the given VMCS info. object.
1962	*
1963	* @param pVCpu The cross context virtual CPU structure.
1964	* @param pVmcsInfo The VMCS info. object.
1965	*
1966	* @remarks No-long-jump zone!!!
1967	*/
1968	static void vmxHCEnableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1969	{
1970	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
1971	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
1972	{
1973	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1974	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
1975	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
1976	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
1977	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
1978	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
1979	Log4Func(("Enabled\n"));
1980	}
1981	}
1982
1983
1984	/**
1985	* Disables VMCS shadowing for the given VMCS info. object.
1986	*
1987	* @param pVCpu The cross context virtual CPU structure.
1988	* @param pVmcsInfo The VMCS info. object.
1989	*
1990	* @remarks No-long-jump zone!!!
1991	*/
1992	static void vmxHCDisableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1993	{
1994	/*
1995	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
1996	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
1997	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
1998	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
1999	*
2000	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
2001	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
2002	*/
2003	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
2004	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
2005	{
2006	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
2007	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
2008	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
2009	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
2010	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
2011	Log4Func(("Disabled\n"));
2012	}
2013	}
2014	#endif
2015
2016
2017	/**
2018	* Exports the guest CR0 control register into the guest-state area in the VMCS.
2019	*
2020	* The guest FPU state is always pre-loaded hence we don't need to bother about
2021	* sharing FPU related CR0 bits between the guest and host.
2022	*
2023	* @returns VBox status code.
2024	* @param pVCpu The cross context virtual CPU structure.
2025	* @param pVmxTransient The VMX-transient structure.
2026	*
2027	* @remarks No-long-jump zone!!!
2028	*/
2029	static int vmxHCExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2030	{
2031	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR0)
2032	{
2033	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2034	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2035
2036	uint64_t fSetCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed0;
2037	uint64_t const fZapCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed1;
2038	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2039	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
2040	else
2041	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2042
2043	if (!pVmxTransient->fIsNestedGuest)
2044	{
2045	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
2046	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
2047	uint64_t const u64ShadowCr0 = u64GuestCr0;
2048	Assert(!RT_HI_U32(u64GuestCr0));
2049
2050	/*
2051	* Setup VT-x's view of the guest CR0.
2052	*/
2053	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
2054	if (VM_IS_VMX_NESTED_PAGING(pVM))
2055	{
2056	#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
2057	if (CPUMIsGuestPagingEnabled(pVCpu))
2058	{
2059	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
2060	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
2061	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
2062	}
2063	else
2064	{
2065	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
2066	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
2067	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
2068	}
2069
2070	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2071	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2072	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
2073	#endif
2074	}
2075	else
2076	{
2077	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2078	u64GuestCr0 \|= X86_CR0_WP;
2079	}
2080
2081	/*
2082	* Guest FPU bits.
2083	*
2084	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
2085	* using CR0.TS.
2086	*
2087	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
2088	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2089	*/
2090	u64GuestCr0 \|= X86_CR0_NE;
2091
2092	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
2093	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
2094
2095	/*
2096	* Update exception intercepts.
2097	*/
2098	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
2099	#ifndef IN_NEM_DARWIN
2100	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2101	{
2102	Assert(PDMVmmDevHeapIsEnabled(pVM));
2103	Assert(pVM->hm.s.vmx.pRealModeTSS);
2104	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2105	}
2106	else
2107	#endif
2108	{
2109	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
2110	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2111	if (fInterceptMF)
2112	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
2113	}
2114
2115	/* Additional intercepts for debugging, define these yourself explicitly. */
2116	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2117	uXcptBitmap \|= 0
2118	\| RT_BIT(X86_XCPT_BP)
2119	\| RT_BIT(X86_XCPT_DE)
2120	\| RT_BIT(X86_XCPT_NM)
2121	\| RT_BIT(X86_XCPT_TS)
2122	\| RT_BIT(X86_XCPT_UD)
2123	\| RT_BIT(X86_XCPT_NP)
2124	\| RT_BIT(X86_XCPT_SS)
2125	\| RT_BIT(X86_XCPT_GP)
2126	\| RT_BIT(X86_XCPT_PF)
2127	\| RT_BIT(X86_XCPT_MF)
2128	;
2129	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2130	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
2131	#endif
2132	if (VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv)
2133	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
2134	if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE)
2135	uXcptBitmap \|= RT_BIT(X86_XCPT_DE);
2136	Assert(VM_IS_VMX_NESTED_PAGING(pVM) \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
2137
2138	/* Apply the hardware specified CR0 fixed bits and enable caching. */
2139	u64GuestCr0 \|= fSetCr0;
2140	u64GuestCr0 &= fZapCr0;
2141	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
2142
2143	Assert(!RT_HI_U32(u64GuestCr0));
2144	Assert(u64GuestCr0 & X86_CR0_NE);
2145
2146	/* Commit the CR0 and related fields to the guest VMCS. */
2147	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
2148	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
2149	if (uProcCtls != pVmcsInfo->u32ProcCtls)
2150	{
2151	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
2152	AssertRC(rc);
2153	}
2154	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
2155	{
2156	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2157	AssertRC(rc);
2158	}
2159
2160	/* Update our caches. */
2161	pVmcsInfo->u32ProcCtls = uProcCtls;
2162	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2163
2164	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
2165	}
2166	else
2167	{
2168	/*
2169	* With nested-guests, we may have extended the guest/host mask here since we
2170	* merged in the outer guest's mask. Thus, the merged mask can include more bits
2171	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
2172	* originally supplied. We must copy those bits from the nested-guest CR0 into
2173	* the nested-guest CR0 read-shadow.
2174	*/
2175	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
2176	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
2177	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
2178
2179	/* Apply the hardware specified CR0 fixed bits and enable caching. */
2180	u64GuestCr0 \|= fSetCr0;
2181	u64GuestCr0 &= fZapCr0;
2182	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
2183
2184	Assert(!RT_HI_U32(u64GuestCr0));
2185	Assert(u64GuestCr0 & X86_CR0_NE);
2186
2187	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
2188	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
2189	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
2190
2191	Log4Func(("cr0=%#RX64 shadow=%#RX64 vmcs_read_shw=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0,
2192	pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0ReadShadow.u, fSetCr0, fZapCr0));
2193	}
2194
2195	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR0);
2196	}
2197
2198	return VINF_SUCCESS;
2199	}
2200
2201
2202	/**
2203	* Exports the guest control registers (CR3, CR4) into the guest-state area
2204	* in the VMCS.
2205	*
2206	* @returns VBox strict status code.
2207	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
2208	* without unrestricted guest access and the VMMDev is not presently
2209	* mapped (e.g. EFI32).
2210	*
2211	* @param pVCpu The cross context virtual CPU structure.
2212	* @param pVmxTransient The VMX-transient structure.
2213	*
2214	* @remarks No-long-jump zone!!!
2215	*/
2216	static VBOXSTRICTRC vmxHCExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2217	{
2218	int rc = VINF_SUCCESS;
2219	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2220
2221	/*
2222	* Guest CR2.
2223	* It's always loaded in the assembler code. Nothing to do here.
2224	*/
2225
2226	/*
2227	* Guest CR3.
2228	*/
2229	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR3)
2230	{
2231	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
2232
2233	if (VM_IS_VMX_NESTED_PAGING(pVM))
2234	{
2235	#ifndef IN_NEM_DARWIN
2236	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2237	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2238
2239	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2240	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
2241	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2242	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
2243
2244	/* VMX_EPT_MEMTYPE_WB support is already checked in vmxHCSetupTaggedTlb(). */
2245	pVmcsInfo->HCPhysEPTP \|= RT_BF_MAKE(VMX_BF_EPTP_MEMTYPE, VMX_EPTP_MEMTYPE_WB)
2246	\| RT_BF_MAKE(VMX_BF_EPTP_PAGE_WALK_LENGTH, VMX_EPTP_PAGE_WALK_LENGTH_4);
2247
2248	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2249	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2250	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
2251	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
2252	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
2253	\|\| (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY),
2254	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
2255
2256	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
2257	AssertRC(rc);
2258	#endif
2259
2260	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2261	uint64_t u64GuestCr3 = pCtx->cr3;
2262	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
2263	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2264	{
2265	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2266	if (CPUMIsGuestInPAEModeEx(pCtx))
2267	{
2268	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, pCtx->aPaePdpes[0].u); AssertRC(rc);
2269	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, pCtx->aPaePdpes[1].u); AssertRC(rc);
2270	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, pCtx->aPaePdpes[2].u); AssertRC(rc);
2271	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, pCtx->aPaePdpes[3].u); AssertRC(rc);
2272	}
2273
2274	/*
2275	* The guest's view of its CR3 is unblemished with nested paging when the
2276	* guest is using paging or we have unrestricted guest execution to handle
2277	* the guest when it's not using paging.
2278	*/
2279	}
2280	#ifndef IN_NEM_DARWIN
2281	else
2282	{
2283	/*
2284	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
2285	* thinks it accesses physical memory directly, we use our identity-mapped
2286	* page table to map guest-linear to guest-physical addresses. EPT takes care
2287	* of translating it to host-physical addresses.
2288	*/
2289	RTGCPHYS GCPhys;
2290	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2291
2292	/* We obtain it here every time as the guest could have relocated this PCI region. */
2293	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2294	if (RT_SUCCESS(rc))
2295	{ /* likely */ }
2296	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
2297	{
2298	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
2299	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
2300	}
2301	else
2302	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
2303
2304	u64GuestCr3 = GCPhys;
2305	}
2306	#endif
2307
2308	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
2309	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, u64GuestCr3);
2310	AssertRC(rc);
2311	}
2312	else
2313	{
2314	Assert(!pVmxTransient->fIsNestedGuest);
2315	/* Non-nested paging case, just use the hypervisor's CR3. */
2316	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
2317
2318	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
2319	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
2320	AssertRC(rc);
2321	}
2322
2323	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR3);
2324	}
2325
2326	/*
2327	* Guest CR4.
2328	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
2329	*/
2330	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR4)
2331	{
2332	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2333	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2334
2335	uint64_t const fSetCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed0;
2336	uint64_t const fZapCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed1;
2337
2338	/*
2339	* With nested-guests, we may have extended the guest/host mask here (since we
2340	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
2341	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
2342	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
2343	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
2344	*/
2345	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
2346	uint64_t u64GuestCr4 = pCtx->cr4;
2347	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
2348	? pCtx->cr4
2349	: CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
2350	Assert(!RT_HI_U32(u64GuestCr4));
2351
2352	#ifndef IN_NEM_DARWIN
2353	/*
2354	* Setup VT-x's view of the guest CR4.
2355	*
2356	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
2357	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
2358	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2359	*
2360	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2361	*/
2362	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2363	{
2364	Assert(pVM->hm.s.vmx.pRealModeTSS);
2365	Assert(PDMVmmDevHeapIsEnabled(pVM));
2366	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
2367	}
2368	#endif
2369
2370	if (VM_IS_VMX_NESTED_PAGING(pVM))
2371	{
2372	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2373	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2374	{
2375	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2376	u64GuestCr4 \|= X86_CR4_PSE;
2377	/* Our identity mapping is a 32-bit page directory. */
2378	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
2379	}
2380	/* else use guest CR4.*/
2381	}
2382	else
2383	{
2384	Assert(!pVmxTransient->fIsNestedGuest);
2385
2386	/*
2387	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2388	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2389	*/
2390	switch (VCPU_2_VMXSTATE(pVCpu).enmShadowMode)
2391	{
2392	case PGMMODE_REAL: /* Real-mode. */
2393	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2394	case PGMMODE_32_BIT: /* 32-bit paging. */
2395	{
2396	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
2397	break;
2398	}
2399
2400	case PGMMODE_PAE: /* PAE paging. */
2401	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2402	{
2403	u64GuestCr4 \|= X86_CR4_PAE;
2404	break;
2405	}
2406
2407	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2408	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2409	{
2410	#ifdef VBOX_WITH_64_BITS_GUESTS
2411	/* For our assumption in vmxHCShouldSwapEferMsr. */
2412	Assert(u64GuestCr4 & X86_CR4_PAE);
2413	break;
2414	#endif
2415	}
2416	default:
2417	AssertFailed();
2418	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2419	}
2420	}
2421
2422	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
2423	u64GuestCr4 \|= fSetCr4;
2424	u64GuestCr4 &= fZapCr4;
2425
2426	Assert(!RT_HI_U32(u64GuestCr4));
2427	Assert(u64GuestCr4 & X86_CR4_VMXE);
2428
2429	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
2430	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
2431	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
2432
2433	#ifndef IN_NEM_DARWIN
2434	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
2435	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
2436	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
2437	{
2438	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
2439	hmR0VmxUpdateStartVmFunction(pVCpu);
2440	}
2441	#endif
2442
2443	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR4);
2444
2445	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
2446	}
2447	return rc;
2448	}
2449
2450
2451	#ifdef VBOX_STRICT
2452	/**
2453	* Strict function to validate segment registers.
2454	*
2455	* @param pVCpu The cross context virtual CPU structure.
2456	* @param pVmcsInfo The VMCS info. object.
2457	*
2458	* @remarks Will import guest CR0 on strict builds during validation of
2459	* segments.
2460	*/
2461	static void vmxHCValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2462	{
2463	/*
2464	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
2465	*
2466	* The reason we check for attribute value 0 in this function and not just the unusable bit is
2467	* because vmxHCExportGuestSegReg() only updates the VMCS' copy of the value with the
2468	* unusable bit and doesn't change the guest-context value.
2469	*/
2470	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2471	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2472	vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
2473	if ( !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
2474	&& ( !CPUMIsGuestInRealModeEx(pCtx)
2475	&& !CPUMIsGuestInV86ModeEx(pCtx)))
2476	{
2477	/* Protected mode checks */
2478	/* CS */
2479	Assert(pCtx->cs.Attr.n.u1Present);
2480	Assert(!(pCtx->cs.Attr.u & 0xf00));
2481	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
2482	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
2483	\|\| !(pCtx->cs.Attr.n.u1Granularity));
2484	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
2485	\|\| (pCtx->cs.Attr.n.u1Granularity));
2486	/* CS cannot be loaded with NULL in protected mode. */
2487	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
2488	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
2489	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
2490	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
2491	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
2492	else
2493	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
2494	/* SS */
2495	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
2496	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
2497	if ( !(pCtx->cr0 & X86_CR0_PE)
2498	\|\| pCtx->cs.Attr.n.u4Type == 3)
2499	{
2500	Assert(!pCtx->ss.Attr.n.u2Dpl);
2501	}
2502	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
2503	{
2504	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
2505	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
2506	Assert(pCtx->ss.Attr.n.u1Present);
2507	Assert(!(pCtx->ss.Attr.u & 0xf00));
2508	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
2509	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
2510	\|\| !(pCtx->ss.Attr.n.u1Granularity));
2511	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
2512	\|\| (pCtx->ss.Attr.n.u1Granularity));
2513	}
2514	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSegReg(). */
2515	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
2516	{
2517	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2518	Assert(pCtx->ds.Attr.n.u1Present);
2519	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
2520	Assert(!(pCtx->ds.Attr.u & 0xf00));
2521	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
2522	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
2523	\|\| !(pCtx->ds.Attr.n.u1Granularity));
2524	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
2525	\|\| (pCtx->ds.Attr.n.u1Granularity));
2526	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2527	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
2528	}
2529	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
2530	{
2531	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2532	Assert(pCtx->es.Attr.n.u1Present);
2533	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
2534	Assert(!(pCtx->es.Attr.u & 0xf00));
2535	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
2536	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
2537	\|\| !(pCtx->es.Attr.n.u1Granularity));
2538	Assert( !(pCtx->es.u32Limit & 0xfff00000)
2539	\|\| (pCtx->es.Attr.n.u1Granularity));
2540	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2541	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
2542	}
2543	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
2544	{
2545	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2546	Assert(pCtx->fs.Attr.n.u1Present);
2547	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
2548	Assert(!(pCtx->fs.Attr.u & 0xf00));
2549	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
2550	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
2551	\|\| !(pCtx->fs.Attr.n.u1Granularity));
2552	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
2553	\|\| (pCtx->fs.Attr.n.u1Granularity));
2554	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2555	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
2556	}
2557	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
2558	{
2559	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2560	Assert(pCtx->gs.Attr.n.u1Present);
2561	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
2562	Assert(!(pCtx->gs.Attr.u & 0xf00));
2563	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
2564	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
2565	\|\| !(pCtx->gs.Attr.n.u1Granularity));
2566	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
2567	\|\| (pCtx->gs.Attr.n.u1Granularity));
2568	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2569	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
2570	}
2571	/* 64-bit capable CPUs. */
2572	Assert(!RT_HI_U32(pCtx->cs.u64Base));
2573	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
2574	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
2575	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
2576	}
2577	else if ( CPUMIsGuestInV86ModeEx(pCtx)
2578	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
2579	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
2580	{
2581	/* Real and v86 mode checks. */
2582	/* vmxHCExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
2583	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
2584	#ifndef IN_NEM_DARWIN
2585	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2586	{
2587	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
2588	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
2589	}
2590	else
2591	#endif
2592	{
2593	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
2594	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
2595	}
2596
2597	/* CS */
2598	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
2599	Assert(pCtx->cs.u32Limit == 0xffff);
2600	AssertMsg(u32CSAttr == 0xf3, ("cs=%#x %#x ", pCtx->cs.Sel, u32CSAttr));
2601	/* SS */
2602	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
2603	Assert(pCtx->ss.u32Limit == 0xffff);
2604	Assert(u32SSAttr == 0xf3);
2605	/* DS */
2606	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
2607	Assert(pCtx->ds.u32Limit == 0xffff);
2608	Assert(u32DSAttr == 0xf3);
2609	/* ES */
2610	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
2611	Assert(pCtx->es.u32Limit == 0xffff);
2612	Assert(u32ESAttr == 0xf3);
2613	/* FS */
2614	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
2615	Assert(pCtx->fs.u32Limit == 0xffff);
2616	Assert(u32FSAttr == 0xf3);
2617	/* GS */
2618	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
2619	Assert(pCtx->gs.u32Limit == 0xffff);
2620	Assert(u32GSAttr == 0xf3);
2621	/* 64-bit capable CPUs. */
2622	Assert(!RT_HI_U32(pCtx->cs.u64Base));
2623	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
2624	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
2625	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
2626	}
2627	}
2628	#endif /* VBOX_STRICT */
2629
2630
2631	/**
2632	* Exports a guest segment register into the guest-state area in the VMCS.
2633	*
2634	* @returns VBox status code.
2635	* @param pVCpu The cross context virtual CPU structure.
2636	* @param pVmcsInfo The VMCS info. object.
2637	* @param iSegReg The segment register number (X86_SREG_XXX).
2638	* @param pSelReg Pointer to the segment selector.
2639	*
2640	* @remarks No-long-jump zone!!!
2641	*/
2642	static int vmxHCExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
2643	{
2644	Assert(iSegReg < X86_SREG_COUNT);
2645
2646	uint32_t u32Access = pSelReg->Attr.u;
2647	#ifndef IN_NEM_DARWIN
2648	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2649	#endif
2650	{
2651	/*
2652	* The way to differentiate between whether this is really a null selector or was just
2653	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
2654	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
2655	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
2656	* NULL selectors loaded in protected-mode have their attribute as 0.
2657	*/
2658	if (u32Access)
2659	{ }
2660	else
2661	u32Access = X86DESCATTR_UNUSABLE;
2662	}
2663	#ifndef IN_NEM_DARWIN
2664	else
2665	{
2666	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
2667	u32Access = 0xf3;
2668	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2669	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2670	RT_NOREF_PV(pVCpu);
2671	}
2672	#else
2673	RT_NOREF(pVmcsInfo);
2674	#endif
2675
2676	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
2677	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
2678	("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));
2679
2680	/*
2681	* Commit it to the VMCS.
2682	*/
2683	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), pSelReg->Sel); AssertRC(rc);
2684	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), pSelReg->u32Limit); AssertRC(rc);
2685	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), pSelReg->u64Base); AssertRC(rc);
2686	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access); AssertRC(rc);
2687	return VINF_SUCCESS;
2688	}
2689
2690
2691	/**
2692	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
2693	* area in the VMCS.
2694	*
2695	* @returns VBox status code.
2696	* @param pVCpu The cross context virtual CPU structure.
2697	* @param pVmxTransient The VMX-transient structure.
2698	*
2699	* @remarks Will import guest CR0 on strict builds during validation of
2700	* segments.
2701	* @remarks No-long-jump zone!!!
2702	*/
2703	static int vmxHCExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2704	{
2705	int rc = VERR_INTERNAL_ERROR_5;
2706	#ifndef IN_NEM_DARWIN
2707	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2708	#endif
2709	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2710	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2711	#ifndef IN_NEM_DARWIN
2712	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
2713	#endif
2714
2715	/*
2716	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
2717	*/
2718	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
2719	{
2720	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CS)
2721	{
2722	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
2723	#ifndef IN_NEM_DARWIN
2724	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2725	pVmcsInfoShared->RealMode.AttrCS.u = pCtx->cs.Attr.u;
2726	#endif
2727	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
2728	AssertRC(rc);
2729	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CS);
2730	}
2731
2732	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SS)
2733	{
2734	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
2735	#ifndef IN_NEM_DARWIN
2736	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2737	pVmcsInfoShared->RealMode.AttrSS.u = pCtx->ss.Attr.u;
2738	#endif
2739	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
2740	AssertRC(rc);
2741	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SS);
2742	}
2743
2744	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_DS)
2745	{
2746	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
2747	#ifndef IN_NEM_DARWIN
2748	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2749	pVmcsInfoShared->RealMode.AttrDS.u = pCtx->ds.Attr.u;
2750	#endif
2751	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
2752	AssertRC(rc);
2753	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_DS);
2754	}
2755
2756	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_ES)
2757	{
2758	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
2759	#ifndef IN_NEM_DARWIN
2760	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2761	pVmcsInfoShared->RealMode.AttrES.u = pCtx->es.Attr.u;
2762	#endif
2763	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
2764	AssertRC(rc);
2765	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_ES);
2766	}
2767
2768	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_FS)
2769	{
2770	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
2771	#ifndef IN_NEM_DARWIN
2772	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2773	pVmcsInfoShared->RealMode.AttrFS.u = pCtx->fs.Attr.u;
2774	#endif
2775	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
2776	AssertRC(rc);
2777	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_FS);
2778	}
2779
2780	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GS)
2781	{
2782	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
2783	#ifndef IN_NEM_DARWIN
2784	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2785	pVmcsInfoShared->RealMode.AttrGS.u = pCtx->gs.Attr.u;
2786	#endif
2787	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
2788	AssertRC(rc);
2789	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GS);
2790	}
2791
2792	#ifdef VBOX_STRICT
2793	vmxHCValidateSegmentRegs(pVCpu, pVmcsInfo);
2794	#endif
2795	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
2796	pCtx->cs.Attr.u));
2797	}
2798
2799	/*
2800	* Guest TR.
2801	*/
2802	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_TR)
2803	{
2804	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
2805
2806	/*
2807	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
2808	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
2809	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
2810	*/
2811	uint16_t u16Sel;
2812	uint32_t u32Limit;
2813	uint64_t u64Base;
2814	uint32_t u32AccessRights;
2815	#ifndef IN_NEM_DARWIN
2816	if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
2817	#endif
2818	{
2819	u16Sel = pCtx->tr.Sel;
2820	u32Limit = pCtx->tr.u32Limit;
2821	u64Base = pCtx->tr.u64Base;
2822	u32AccessRights = pCtx->tr.Attr.u;
2823	}
2824	#ifndef IN_NEM_DARWIN
2825	else
2826	{
2827	Assert(!pVmxTransient->fIsNestedGuest);
2828	Assert(pVM->hm.s.vmx.pRealModeTSS);
2829	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
2830
2831	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
2832	RTGCPHYS GCPhys;
2833	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
2834	AssertRCReturn(rc, rc);
2835
2836	X86DESCATTR DescAttr;
2837	DescAttr.u = 0;
2838	DescAttr.n.u1Present = 1;
2839	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2840
2841	u16Sel = 0;
2842	u32Limit = HM_VTX_TSS_SIZE;
2843	u64Base = GCPhys;
2844	u32AccessRights = DescAttr.u;
2845	}
2846	#endif
2847
2848	/* Validate. */
2849	Assert(!(u16Sel & RT_BIT(2)));
2850	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
2851	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
2852	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
2853	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
2854	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
2855	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
2856	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
2857	Assert( (u32Limit & 0xfff) == 0xfff
2858	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
2859	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
2860	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
2861
2862	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
2863	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
2864	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
2865	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
2866
2867	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_TR);
2868	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
2869	}
2870
2871	/*
2872	* Guest GDTR.
2873	*/
2874	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GDTR)
2875	{
2876	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
2877
2878	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
2879	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
2880
2881	/* Validate. */
2882	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
2883
2884	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
2885	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
2886	}
2887
2888	/*
2889	* Guest LDTR.
2890	*/
2891	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_LDTR)
2892	{
2893	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
2894
2895	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
2896	uint32_t u32Access;
2897	if ( !pVmxTransient->fIsNestedGuest
2898	&& !pCtx->ldtr.Attr.u)
2899	u32Access = X86DESCATTR_UNUSABLE;
2900	else
2901	u32Access = pCtx->ldtr.Attr.u;
2902
2903	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
2904	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
2905	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
2906	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
2907
2908	/* Validate. */
2909	if (!(u32Access & X86DESCATTR_UNUSABLE))
2910	{
2911	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
2912	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
2913	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
2914	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
2915	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
2916	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
2917	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
2918	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
2919	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
2920	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
2921	}
2922
2923	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
2924	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
2925	}
2926
2927	/*
2928	* Guest IDTR.
2929	*/
2930	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_IDTR)
2931	{
2932	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
2933
2934	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
2935	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
2936
2937	/* Validate. */
2938	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
2939
2940	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
2941	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
2942	}
2943
2944	return VINF_SUCCESS;
2945	}
2946
2947
2948	/**
2949	* Gets the IEM exception flags for the specified vector and IDT vectoring /
2950	* VM-exit interruption info type.
2951	*
2952	* @returns The IEM exception flags.
2953	* @param uVector The event vector.
2954	* @param uVmxEventType The VMX event type.
2955	*
2956	* @remarks This function currently only constructs flags required for
2957	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
2958	* and CR2 aspects of an exception are not included).
2959	*/
2960	static uint32_t vmxHCGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
2961	{
2962	uint32_t fIemXcptFlags;
2963	switch (uVmxEventType)
2964	{
2965	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
2966	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
2967	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
2968	break;
2969
2970	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
2971	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
2972	break;
2973
2974	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
2975	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
2976	break;
2977
2978	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
2979	{
2980	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
2981	if (uVector == X86_XCPT_BP)
2982	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
2983	else if (uVector == X86_XCPT_OF)
2984	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
2985	else
2986	{
2987	fIemXcptFlags = 0;
2988	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
2989	}
2990	break;
2991	}
2992
2993	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
2994	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
2995	break;
2996
2997	default:
2998	fIemXcptFlags = 0;
2999	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
3000	break;
3001	}
3002	return fIemXcptFlags;
3003	}
3004
3005
3006	/**
3007	* Sets an event as a pending event to be injected into the guest.
3008	*
3009	* @param pVCpu The cross context virtual CPU structure.
3010	* @param u32IntInfo The VM-entry interruption-information field.
3011	* @param cbInstr The VM-entry instruction length in bytes (for
3012	* software interrupts, exceptions and privileged
3013	* software exceptions).
3014	* @param u32ErrCode The VM-entry exception error code.
3015	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3016	* page-fault.
3017	*/
3018	DECLINLINE(void) vmxHCSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
3019	RTGCUINTPTR GCPtrFaultAddress)
3020	{
3021	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
3022	VCPU_2_VMXSTATE(pVCpu).Event.fPending = true;
3023	VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo = u32IntInfo;
3024	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode = u32ErrCode;
3025	VCPU_2_VMXSTATE(pVCpu).Event.cbInstr = cbInstr;
3026	VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress = GCPtrFaultAddress;
3027	}
3028
3029
3030	/**
3031	* Sets an external interrupt as pending-for-injection into the VM.
3032	*
3033	* @param pVCpu The cross context virtual CPU structure.
3034	* @param u8Interrupt The external interrupt vector.
3035	*/
3036	DECLINLINE(void) vmxHCSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
3037	{
3038	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
3039	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
3040	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3041	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3042	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3043	Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
3044	}
3045
3046
3047	/**
3048	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
3049	*
3050	* @param pVCpu The cross context virtual CPU structure.
3051	*/
3052	DECLINLINE(void) vmxHCSetPendingXcptNmi(PVMCPUCC pVCpu)
3053	{
3054	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
3055	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
3056	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3057	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3058	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3059	Log4Func(("NMI pending injection\n"));
3060	}
3061
3062
3063	/**
3064	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
3065	*
3066	* @param pVCpu The cross context virtual CPU structure.
3067	*/
3068	DECLINLINE(void) vmxHCSetPendingXcptDF(PVMCPUCC pVCpu)
3069	{
3070	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
3071	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3072	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3073	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3074	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3075	}
3076
3077
3078	/**
3079	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3080	*
3081	* @param pVCpu The cross context virtual CPU structure.
3082	*/
3083	DECLINLINE(void) vmxHCSetPendingXcptUD(PVMCPUCC pVCpu)
3084	{
3085	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
3086	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3087	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3088	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3089	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3090	}
3091
3092
3093	/**
3094	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3095	*
3096	* @param pVCpu The cross context virtual CPU structure.
3097	*/
3098	DECLINLINE(void) vmxHCSetPendingXcptDB(PVMCPUCC pVCpu)
3099	{
3100	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
3101	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3102	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3103	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3104	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3105	}
3106
3107
3108	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3109	/**
3110	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
3111	*
3112	* @param pVCpu The cross context virtual CPU structure.
3113	* @param u32ErrCode The error code for the general-protection exception.
3114	*/
3115	DECLINLINE(void) vmxHCSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
3116	{
3117	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
3118	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3119	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3120	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3121	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
3122	}
3123
3124
3125	/**
3126	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
3127	*
3128	* @param pVCpu The cross context virtual CPU structure.
3129	* @param u32ErrCode The error code for the stack exception.
3130	*/
3131	DECLINLINE(void) vmxHCSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
3132	{
3133	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
3134	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3135	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3136	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3137	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
3138	}
3139	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3140
3141
3142	/**
3143	* Fixes up attributes for the specified segment register.
3144	*
3145	* @param pVCpu The cross context virtual CPU structure.
3146	* @param pSelReg The segment register that needs fixing.
3147	* @param pszRegName The register name (for logging and assertions).
3148	*/
3149	static void vmxHCFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
3150	{
3151	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
3152
3153	/*
3154	* If VT-x marks the segment as unusable, most other bits remain undefined:
3155	* - For CS the L, D and G bits have meaning.
3156	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
3157	* - For the remaining data segments no bits are defined.
3158	*
3159	* The present bit and the unusable bit has been observed to be set at the
3160	* same time (the selector was supposed to be invalid as we started executing
3161	* a V8086 interrupt in ring-0).
3162	*
3163	* What should be important for the rest of the VBox code, is that the P bit is
3164	* cleared. Some of the other VBox code recognizes the unusable bit, but
3165	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
3166	* safe side here, we'll strip off P and other bits we don't care about. If
3167	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
3168	*
3169	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
3170	*/
3171	#ifdef VBOX_STRICT
3172	uint32_t const uAttr = pSelReg->Attr.u;
3173	#endif
3174
3175	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
3176	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
3177	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
3178
3179	#ifdef VBOX_STRICT
3180	# ifndef IN_NEM_DARWIN
3181	VMMRZCallRing3Disable(pVCpu);
3182	# endif
3183	Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
3184	# ifdef DEBUG_bird
3185	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
3186	("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
3187	pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
3188	# endif
3189	# ifndef IN_NEM_DARWIN
3190	VMMRZCallRing3Enable(pVCpu);
3191	# endif
3192	NOREF(uAttr);
3193	#endif
3194	RT_NOREF2(pVCpu, pszRegName);
3195	}
3196
3197
3198	/**
3199	* Imports a guest segment register from the current VMCS into the guest-CPU
3200	* context.
3201	*
3202	* @param pVCpu The cross context virtual CPU structure.
3203	* @tparam a_iSegReg The segment register number (X86_SREG_XXX).
3204	*
3205	* @remarks Called with interrupts and/or preemption disabled.
3206	*/
3207	template<uint32_t const a_iSegReg>
3208	DECLINLINE(void) vmxHCImportGuestSegReg(PVMCPUCC pVCpu)
3209	{
3210	AssertCompile(a_iSegReg < X86_SREG_COUNT);
3211	/* Check that the macros we depend upon here and in the export parenter function works: */
3212	#define MY_SEG_VMCS_FIELD(a_FieldPrefix, a_FieldSuff) \
3213	( a_iSegReg == X86_SREG_ES ? a_FieldPrefix ## ES ## a_FieldSuff \
3214	: a_iSegReg == X86_SREG_CS ? a_FieldPrefix ## CS ## a_FieldSuff \
3215	: a_iSegReg == X86_SREG_SS ? a_FieldPrefix ## SS ## a_FieldSuff \
3216	: a_iSegReg == X86_SREG_DS ? a_FieldPrefix ## DS ## a_FieldSuff \
3217	: a_iSegReg == X86_SREG_FS ? a_FieldPrefix ## FS ## a_FieldSuff \
3218	: a_iSegReg == X86_SREG_GS ? a_FieldPrefix ## GS ## a_FieldSuff : 0)
3219	AssertCompile(VMX_VMCS_GUEST_SEG_BASE(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS_GUEST_,_BASE));
3220	AssertCompile(VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS16_GUEST_,_SEL));
3221	AssertCompile(VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_LIMIT));
3222	AssertCompile(VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_ACCESS_RIGHTS));
3223
3224	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[a_iSegReg];
3225
3226	uint16_t u16Sel;
3227	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg), &u16Sel); AssertRC(rc);
3228	pSelReg->Sel = u16Sel;
3229	pSelReg->ValidSel = u16Sel;
3230
3231	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg), &pSelReg->u32Limit); AssertRC(rc);
3232	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(a_iSegReg), &pSelReg->u64Base); AssertRC(rc);
3233
3234	uint32_t u32Attr;
3235	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg), &u32Attr); AssertRC(rc);
3236	pSelReg->Attr.u = u32Attr;
3237	if (u32Attr & X86DESCATTR_UNUSABLE)
3238	vmxHCFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + a_iSegReg * 3);
3239
3240	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
3241	}
3242
3243
3244	/**
3245	* Imports the guest LDTR from the VMCS into the guest-CPU context.
3246	*
3247	* @param pVCpu The cross context virtual CPU structure.
3248	*
3249	* @remarks Called with interrupts and/or preemption disabled.
3250	*/
3251	DECL_FORCE_INLINE(void) vmxHCImportGuestLdtr(PVMCPUCC pVCpu)
3252	{
3253	uint16_t u16Sel;
3254	uint64_t u64Base;
3255	uint32_t u32Limit, u32Attr;
3256	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
3257	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
3258	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
3259	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
3260
3261	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
3262	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
3263	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
3264	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
3265	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
3266	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
3267	if (u32Attr & X86DESCATTR_UNUSABLE)
3268	vmxHCFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
3269	}
3270
3271
3272	/**
3273	* Imports the guest TR from the VMCS into the guest-CPU context.
3274	*
3275	* @param pVCpu The cross context virtual CPU structure.
3276	*
3277	* @remarks Called with interrupts and/or preemption disabled.
3278	*/
3279	DECL_FORCE_INLINE(void) vmxHCImportGuestTr(PVMCPUCC pVCpu)
3280	{
3281	uint16_t u16Sel;
3282	uint64_t u64Base;
3283	uint32_t u32Limit, u32Attr;
3284	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
3285	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
3286	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
3287	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
3288
3289	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
3290	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
3291	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
3292	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
3293	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
3294	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
3295	/* TR is the only selector that can never be unusable. */
3296	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
3297	}
3298
3299
3300	/**
3301	* Core: Imports the guest RIP from the VMCS into the guest-CPU context.
3302	*
3303	* @returns The RIP value.
3304	* @param pVCpu The cross context virtual CPU structure.
3305	*
3306	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3307	* @remarks Do -not- call this function directly!
3308	*/
3309	DECL_FORCE_INLINE(uint64_t) vmxHCImportGuestCoreRip(PVMCPUCC pVCpu)
3310	{
3311	uint64_t u64Val;
3312	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
3313	AssertRC(rc);
3314
3315	pVCpu->cpum.GstCtx.rip = u64Val;
3316
3317	return u64Val;
3318	}
3319
3320
3321	/**
3322	* Imports the guest RIP from the VMCS into the guest-CPU context.
3323	*
3324	* @param pVCpu The cross context virtual CPU structure.
3325	*
3326	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3327	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3328	* instead!!!
3329	*/
3330	DECL_FORCE_INLINE(void) vmxHCImportGuestRip(PVMCPUCC pVCpu)
3331	{
3332	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP)
3333	{
3334	EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false);
3335	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
3336	}
3337	}
3338
3339
3340	/**
3341	* Core: Imports the guest RFLAGS from the VMCS into the guest-CPU context.
3342	*
3343	* @param pVCpu The cross context virtual CPU structure.
3344	* @param pVmcsInfo The VMCS info. object.
3345	*
3346	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3347	* @remarks Do -not- call this function directly!
3348	*/
3349	DECL_FORCE_INLINE(void) vmxHCImportGuestCoreRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3350	{
3351	uint64_t fRFlags;
3352	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &fRFlags);
3353	AssertRC(rc);
3354
3355	Assert((fRFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK);
3356	Assert((fRFlags & ~(uint64_t)(X86_EFL_1 \| X86_EFL_LIVE_MASK)) == 0);
3357
3358	pVCpu->cpum.GstCtx.rflags.u = fRFlags;
3359	#ifndef IN_NEM_DARWIN
3360	PCVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3361	if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
3362	{ /* mostly likely */ }
3363	else
3364	{
3365	pVCpu->cpum.GstCtx.eflags.Bits.u1VM = 0;
3366	pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL = pVmcsInfoShared->RealMode.Eflags.Bits.u2IOPL;
3367	}
3368	#else
3369	RT_NOREF(pVmcsInfo);
3370	#endif
3371	}
3372
3373
3374	/**
3375	* Imports the guest RFLAGS from the VMCS into the guest-CPU context.
3376	*
3377	* @param pVCpu The cross context virtual CPU structure.
3378	* @param pVmcsInfo The VMCS info. object.
3379	*
3380	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3381	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3382	* instead!!!
3383	*/
3384	DECL_FORCE_INLINE(void) vmxHCImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3385	{
3386	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RFLAGS)
3387	{
3388	vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo);
3389	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
3390	}
3391	}
3392
3393
3394	#ifndef IN_NEM_DARWIN
3395	/**
3396	* Imports the guest TSX AUX and certain other MSRs from the VMCS into the guest-CPU
3397	* context.
3398	*
3399	* The other MSRs are in the VM-exit MSR-store.
3400	*
3401	* @returns VBox status code.
3402	* @param pVCpu The cross context virtual CPU structure.
3403	* @param pVmcsInfo The VMCS info. object.
3404	* @param fEFlags Saved EFLAGS for restoring the interrupt flag (in case of
3405	* unexpected errors). Ignored in NEM/darwin context.
3406	*/
3407	DECL_FORCE_INLINE(int) vmxHCImportGuestTscAuxAndOtherMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t fEFlags)
3408	{
3409	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3410	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
3411	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
3412	Assert(pMsrs);
3413	Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
3414	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
3415	for (uint32_t i = 0; i < cMsrs; i++)
3416	{
3417	uint32_t const idMsr = pMsrs[i].u32Msr;
3418	switch (idMsr)
3419	{
3420	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
3421	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
3422	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
3423	default:
3424	{
3425	uint32_t idxLbrMsr;
3426	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3427	if (VM_IS_VMX_LBR(pVM))
3428	{
3429	if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
3430	{
3431	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
3432	pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
3433	break;
3434	}
3435	if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
3436	{
3437	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
3438	pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
3439	break;
3440	}
3441	if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
3442	{
3443	pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
3444	break;
3445	}
3446	/* Fallthru (no break) */
3447	}
3448	pVCpu->cpum.GstCtx.fExtrn = 0;
3449	VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
3450	ASMSetFlags(fEFlags);
3451	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
3452	return VERR_HM_UNEXPECTED_LD_ST_MSR;
3453	}
3454	}
3455	}
3456	return VINF_SUCCESS;
3457	}
3458	#endif /* !IN_NEM_DARWIN */
3459
3460
3461	/**
3462	* Imports the guest CR0 from the VMCS into the guest-CPU context.
3463	*
3464	* @param pVCpu The cross context virtual CPU structure.
3465	* @param pVmcsInfo The VMCS info. object.
3466	*/
3467	DECL_FORCE_INLINE(void) vmxHCImportGuestCr0(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3468	{
3469	uint64_t u64Cr0;
3470	uint64_t u64Shadow;
3471	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
3472	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
3473	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
3474	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3475	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
3476	#else
3477	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
3478	{
3479	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3480	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
3481	}
3482	else
3483	{
3484	/*
3485	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
3486	* the nested-guest using hardware-assisted VMX. Accordingly we need to
3487	* re-construct CR0. See @bugref{9180#c95} for details.
3488	*/
3489	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
3490	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3491	u64Cr0 = (u64Cr0 & ~(pVmcsInfoGst->u64Cr0Mask & pVmcsNstGst->u64Cr0Mask.u))
3492	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
3493	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
3494	Assert(u64Cr0 & X86_CR0_NE);
3495	}
3496	#endif
3497
3498	#ifndef IN_NEM_DARWIN
3499	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
3500	#endif
3501	CPUMSetGuestCR0(pVCpu, u64Cr0);
3502	#ifndef IN_NEM_DARWIN
3503	VMMRZCallRing3Enable(pVCpu);
3504	#endif
3505	}
3506
3507
3508	/**
3509	* Imports the guest CR3 from the VMCS into the guest-CPU context.
3510	*
3511	* @param pVCpu The cross context virtual CPU structure.
3512	*/
3513	DECL_FORCE_INLINE(void) vmxHCImportGuestCr3(PVMCPUCC pVCpu)
3514	{
3515	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3516	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3517
3518	/* CR0.PG bit changes are always intercepted, so it's up to date. */
3519	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
3520	\|\| ( VM_IS_VMX_NESTED_PAGING(pVM)
3521	&& CPUMIsGuestPagingEnabledEx(pCtx)))
3522	{
3523	uint64_t u64Cr3;
3524	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
3525	if (pCtx->cr3 != u64Cr3)
3526	{
3527	pCtx->cr3 = u64Cr3;
3528	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
3529	}
3530
3531	/*
3532	* If the guest is in PAE mode, sync back the PDPE's into the guest state.
3533	* CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
3534	*/
3535	if (CPUMIsGuestInPAEModeEx(pCtx))
3536	{
3537	X86PDPE aPaePdpes[4];
3538	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc);
3539	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc);
3540	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc);
3541	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc);
3542	if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes)))
3543	{
3544	memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
3545	/* PGM now updates PAE PDPTEs while updating CR3. */
3546	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
3547	}
3548	}
3549	}
3550	}
3551
3552
3553	/**
3554	* Imports the guest CR4 from the VMCS into the guest-CPU context.
3555	*
3556	* @param pVCpu The cross context virtual CPU structure.
3557	* @param pVmcsInfo The VMCS info. object.
3558	*/
3559	DECL_FORCE_INLINE(void) vmxHCImportGuestCr4(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3560	{
3561	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3562	uint64_t u64Cr4;
3563	uint64_t u64Shadow;
3564	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
3565	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
3566	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
3567	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3568	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
3569	#else
3570	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
3571	{
3572	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3573	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
3574	}
3575	else
3576	{
3577	/*
3578	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
3579	* the nested-guest using hardware-assisted VMX. Accordingly we need to
3580	* re-construct CR4. See @bugref{9180#c95} for details.
3581	*/
3582	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
3583	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3584	u64Cr4 = (u64Cr4 & ~(pVmcsInfo->u64Cr4Mask & pVmcsNstGst->u64Cr4Mask.u))
3585	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
3586	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
3587	Assert(u64Cr4 & X86_CR4_VMXE);
3588	}
3589	#endif
3590	pCtx->cr4 = u64Cr4;
3591	}
3592
3593
3594	/**
3595	* Worker for vmxHCImportGuestIntrState that handles the case where any of the
3596	* relevant VMX_VMCS32_GUEST_INT_STATE bits are set.
3597	*/
3598	DECL_NO_INLINE(static,void) vmxHCImportGuestIntrStateSlow(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t fGstIntState)
3599	{
3600	/*
3601	* We must import RIP here to set our EM interrupt-inhibited state.
3602	* We also import RFLAGS as our code that evaluates pending interrupts
3603	* before VM-entry requires it.
3604	*/
3605	vmxHCImportGuestRip(pVCpu);
3606	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
3607
3608	CPUMUpdateInterruptShadowSsStiEx(&pVCpu->cpum.GstCtx,
3609	RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
3610	RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
3611	pVCpu->cpum.GstCtx.rip);
3612	CPUMUpdateInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx, RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
3613	}
3614
3615
3616	/**
3617	* Imports the guest interruptibility-state from the VMCS into the guest-CPU
3618	* context.
3619	*
3620	* @note May import RIP and RFLAGS if interrupt or NMI are blocked.
3621	*
3622	* @param pVCpu The cross context virtual CPU structure.
3623	* @param pVmcsInfo The VMCS info. object.
3624	*
3625	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
3626	* do not log!
3627	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3628	* instead!!!
3629	*/
3630	DECLINLINE(void) vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3631	{
3632	uint32_t u32Val;
3633	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
3634	Assert((u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
3635	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
3636	if (!u32Val)
3637	{
3638	CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
3639	CPUMClearInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
3640	}
3641	else
3642	vmxHCImportGuestIntrStateSlow(pVCpu, pVmcsInfo, u32Val);
3643	}
3644
3645
3646	/**
3647	* Worker for VMXR0ImportStateOnDemand.
3648	*
3649	* @returns VBox status code.
3650	* @param pVCpu The cross context virtual CPU structure.
3651	* @param pVmcsInfo The VMCS info. object.
3652	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
3653	*/
3654	static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
3655	{
3656	int rc = VINF_SUCCESS;
3657	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3658	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3659	uint32_t u32Val;
3660
3661	/*
3662	* Note! This is hack to workaround a mysterious BSOD observed with release builds
3663	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
3664	* neither are other host platforms.
3665	*
3666	* Committing this temporarily as it prevents BSOD.
3667	*
3668	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
3669	*/
3670	#ifdef RT_OS_WINDOWS
3671	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
3672	return VERR_HM_IPE_1;
3673	#endif
3674
3675	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3676
3677	#ifndef IN_NEM_DARWIN
3678	/*
3679	* We disable interrupts to make the updating of the state and in particular
3680	* the fExtrn modification atomic wrt to preemption hooks.
3681	*/
3682	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
3683	#endif
3684
3685	fWhat &= pCtx->fExtrn;
3686	if (fWhat)
3687	{
3688	do
3689	{
3690	if (fWhat & CPUMCTX_EXTRN_RIP)
3691	vmxHCImportGuestRip(pVCpu);
3692
3693	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
3694	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
3695
3696	/* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */
3697	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
3698	vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
3699
3700	if (fWhat & CPUMCTX_EXTRN_RSP)
3701	{
3702	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pCtx->rsp);
3703	AssertRC(rc);
3704	}
3705
3706	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
3707	{
3708	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3709	#ifndef IN_NEM_DARWIN
3710	bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
3711	#else
3712	bool const fRealOnV86Active = false; /* HV supports only unrestricted guest execution. */
3713	#endif
3714	if (fWhat & CPUMCTX_EXTRN_CS)
3715	{
3716	vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
3717	vmxHCImportGuestRip(pVCpu); /** @todo WTF? */
3718	if (fRealOnV86Active)
3719	pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
3720	EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
3721	}
3722	if (fWhat & CPUMCTX_EXTRN_SS)
3723	{
3724	vmxHCImportGuestSegReg<X86_SREG_SS>(pVCpu);
3725	if (fRealOnV86Active)
3726	pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
3727	}
3728	if (fWhat & CPUMCTX_EXTRN_DS)
3729	{
3730	vmxHCImportGuestSegReg<X86_SREG_DS>(pVCpu);
3731	if (fRealOnV86Active)
3732	pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
3733	}
3734	if (fWhat & CPUMCTX_EXTRN_ES)
3735	{
3736	vmxHCImportGuestSegReg<X86_SREG_ES>(pVCpu);
3737	if (fRealOnV86Active)
3738	pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
3739	}
3740	if (fWhat & CPUMCTX_EXTRN_FS)
3741	{
3742	vmxHCImportGuestSegReg<X86_SREG_FS>(pVCpu);
3743	if (fRealOnV86Active)
3744	pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
3745	}
3746	if (fWhat & CPUMCTX_EXTRN_GS)
3747	{
3748	vmxHCImportGuestSegReg<X86_SREG_GS>(pVCpu);
3749	if (fRealOnV86Active)
3750	pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
3751	}
3752	}
3753
3754	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
3755	{
3756	if (fWhat & CPUMCTX_EXTRN_LDTR)
3757	vmxHCImportGuestLdtr(pVCpu);
3758
3759	if (fWhat & CPUMCTX_EXTRN_GDTR)
3760	{
3761	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
3762	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
3763	pCtx->gdtr.cbGdt = u32Val;
3764	}
3765
3766	/* Guest IDTR. */
3767	if (fWhat & CPUMCTX_EXTRN_IDTR)
3768	{
3769	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
3770	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
3771	pCtx->idtr.cbIdt = u32Val;
3772	}
3773
3774	/* Guest TR. */
3775	if (fWhat & CPUMCTX_EXTRN_TR)
3776	{
3777	#ifndef IN_NEM_DARWIN
3778	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
3779	don't need to import that one. */
3780	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
3781	#endif
3782	vmxHCImportGuestTr(pVCpu);
3783	}
3784	}
3785
3786	if (fWhat & CPUMCTX_EXTRN_DR7)
3787	{
3788	#ifndef IN_NEM_DARWIN
3789	if (!pVCpu->hmr0.s.fUsingHyperDR7)
3790	#endif
3791	{
3792	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);
3793	AssertRC(rc);
3794	}
3795	}
3796
3797	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
3798	{
3799	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
3800	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
3801	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
3802	pCtx->SysEnter.cs = u32Val;
3803	}
3804
3805	#ifndef IN_NEM_DARWIN
3806	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
3807	{
3808	if ( pVM->hmr0.s.fAllow64BitGuests
3809	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
3810	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
3811	}
3812
3813	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
3814	{
3815	if ( pVM->hmr0.s.fAllow64BitGuests
3816	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
3817	{
3818	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3819	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
3820	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
3821	}
3822	}
3823
3824	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
3825	{
3826	rc = vmxHCImportGuestTscAuxAndOtherMsrs(pVCpu, pVmcsInfo, fEFlags);
3827	AssertRCReturn(rc, rc);
3828	}
3829	#else
3830	NOREF(pVM);
3831	#endif
3832
3833	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
3834	{
3835	if (fWhat & CPUMCTX_EXTRN_CR0)
3836	vmxHCImportGuestCr0(pVCpu, pVmcsInfo);
3837
3838	if (fWhat & CPUMCTX_EXTRN_CR4)
3839	vmxHCImportGuestCr4(pVCpu, pVmcsInfo);
3840
3841	if (fWhat & CPUMCTX_EXTRN_CR3)
3842	vmxHCImportGuestCr3(pVCpu);
3843	}
3844
3845	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3846	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
3847	{
3848	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
3849	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
3850	{
3851	Assert(CPUMIsGuestInVmxRootMode(pCtx));
3852	rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
3853	if (RT_SUCCESS(rc))
3854	{ /* likely */ }
3855	else
3856	break;
3857	}
3858	}
3859	#endif
3860	} while (0);
3861
3862	if (RT_SUCCESS(rc))
3863	{
3864	/* Update fExtrn. */
3865	pCtx->fExtrn &= ~fWhat;
3866
3867	/* If everything has been imported, clear the HM keeper bit. */
3868	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
3869	{
3870	#ifndef IN_NEM_DARWIN
3871	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
3872	#else
3873	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
3874	#endif
3875	Assert(!pCtx->fExtrn);
3876	}
3877	}
3878	}
3879	#ifndef IN_NEM_DARWIN
3880	else
3881	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
3882
3883	/*
3884	* Restore interrupts.
3885	*/
3886	ASMSetFlags(fEFlags);
3887	#endif
3888
3889	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3890
3891	if (RT_SUCCESS(rc))
3892	{ /* likely */ }
3893	else
3894	return rc;
3895
3896	/*
3897	* Honor any pending CR3 updates.
3898	*
3899	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
3900	* -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
3901	* -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
3902	*
3903	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
3904	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
3905	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
3906	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
3907	*
3908	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
3909	*
3910	* The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
3911	*/
3912	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)
3913	#ifndef IN_NEM_DARWIN
3914	&& VMMRZCallRing3IsEnabled(pVCpu)
3915	#endif
3916	)
3917	{
3918	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
3919	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3920	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
3921	}
3922
3923	return VINF_SUCCESS;
3924	}
3925
3926
3927	/**
3928	* Internal state fetcher, inner version where we fetch all of a_fWhat.
3929	*
3930	* @returns VBox status code.
3931	* @param pVCpu The cross context virtual CPU structure.
3932	* @param pVmcsInfo The VMCS info. object.
3933	* @param fEFlags Saved EFLAGS for restoring the interrupt flag. Ignored
3934	* in NEM/darwin context.
3935	* @tparam a_fWhat What to import, zero or more bits from
3936	* HMVMX_CPUMCTX_EXTRN_ALL.
3937	*/
3938	template<uint64_t const a_fWhat>
3939	static int vmxHCImportGuestStateInner(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t fEFlags)
3940	{
3941	Assert(a_fWhat != 0); /* No AssertCompile as the assertion probably kicks in before the compiler (clang) discards it. */
3942	AssertCompile(!(a_fWhat & ~HMVMX_CPUMCTX_EXTRN_ALL));
3943	Assert( (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == a_fWhat
3944	\|\| (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == (a_fWhat & ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)));
3945
3946	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3947
3948	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3949
3950	/* RIP and RFLAGS may have been imported already by the post exit code
3951	together with the CPUMCTX_EXTRN_INHIBIT_INT/NMI state, so this part
3952	of the code is skipping this part of the code. */
3953	if ( (a_fWhat & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS))
3954	&& pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS))
3955	{
3956	if (a_fWhat & CPUMCTX_EXTRN_RFLAGS)
3957	vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo);
3958
3959	if (a_fWhat & CPUMCTX_EXTRN_RIP)
3960	{
3961	if (!(a_fWhat & CPUMCTX_EXTRN_CS))
3962	EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false);
3963	else
3964	vmxHCImportGuestCoreRip(pVCpu);
3965	}
3966	}
3967
3968	/* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */
3969	if (a_fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
3970	vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
3971
3972	if (a_fWhat & (CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TR))
3973	{
3974	if (a_fWhat & CPUMCTX_EXTRN_CS)
3975	{
3976	vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
3977	/** @todo try get rid of this carp, it smells and is probably never ever
3978	* used: */
3979	if ( !(a_fWhat & CPUMCTX_EXTRN_RIP)
3980	&& (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP))
3981	{
3982	vmxHCImportGuestCoreRip(pVCpu);
3983	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
3984	}
3985	EMHistoryUpdatePC(pVCpu, pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, true /* fFlattened */);
3986	}
3987	if (a_fWhat & CPUMCTX_EXTRN_SS)
3988	vmxHCImportGuestSegReg<X86_SREG_SS>(pVCpu);
3989	if (a_fWhat & CPUMCTX_EXTRN_DS)
3990	vmxHCImportGuestSegReg<X86_SREG_DS>(pVCpu);
3991	if (a_fWhat & CPUMCTX_EXTRN_ES)
3992	vmxHCImportGuestSegReg<X86_SREG_ES>(pVCpu);
3993	if (a_fWhat & CPUMCTX_EXTRN_FS)
3994	vmxHCImportGuestSegReg<X86_SREG_FS>(pVCpu);
3995	if (a_fWhat & CPUMCTX_EXTRN_GS)
3996	vmxHCImportGuestSegReg<X86_SREG_GS>(pVCpu);
3997
3998	/* Guest TR.
3999	Real-mode emulation using virtual-8086 mode has the fake TSS
4000	(pRealModeTSS) in TR, don't need to import that one. */
4001	#ifndef IN_NEM_DARWIN
4002	PVMXVMCSINFOSHARED const pVmcsInfoShared = pVmcsInfo->pShared;
4003	bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
4004	if ((a_fWhat & CPUMCTX_EXTRN_TR) && !fRealOnV86Active)
4005	#else
4006	if (a_fWhat & CPUMCTX_EXTRN_TR)
4007	#endif
4008	vmxHCImportGuestTr(pVCpu);
4009
4010	#ifndef IN_NEM_DARWIN /* NEM/Darwin: HV supports only unrestricted guest execution. */
4011	if (fRealOnV86Active)
4012	{
4013	if (a_fWhat & CPUMCTX_EXTRN_CS)
4014	pVCpu->cpum.GstCtx.cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
4015	if (a_fWhat & CPUMCTX_EXTRN_SS)
4016	pVCpu->cpum.GstCtx.ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
4017	if (a_fWhat & CPUMCTX_EXTRN_DS)
4018	pVCpu->cpum.GstCtx.ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
4019	if (a_fWhat & CPUMCTX_EXTRN_ES)
4020	pVCpu->cpum.GstCtx.es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
4021	if (a_fWhat & CPUMCTX_EXTRN_FS)
4022	pVCpu->cpum.GstCtx.fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
4023	if (a_fWhat & CPUMCTX_EXTRN_GS)
4024	pVCpu->cpum.GstCtx.gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
4025	}
4026	#endif
4027	}
4028
4029	if (a_fWhat & CPUMCTX_EXTRN_RSP)
4030	{
4031	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pVCpu->cpum.GstCtx.rsp);
4032	AssertRC(rc);
4033	}
4034
4035	if (a_fWhat & CPUMCTX_EXTRN_LDTR)
4036	vmxHCImportGuestLdtr(pVCpu);
4037
4038	if (a_fWhat & CPUMCTX_EXTRN_GDTR)
4039	{
4040	int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pVCpu->cpum.GstCtx.gdtr.pGdt); AssertRC(rc1);
4041	uint32_t u32Val;
4042	int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc2);
4043	pVCpu->cpum.GstCtx.gdtr.cbGdt = (uint16_t)u32Val;
4044	}
4045
4046	/* Guest IDTR. */
4047	if (a_fWhat & CPUMCTX_EXTRN_IDTR)
4048	{
4049	int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pVCpu->cpum.GstCtx.idtr.pIdt); AssertRC(rc1);
4050	uint32_t u32Val;
4051	int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc2);
4052	pVCpu->cpum.GstCtx.idtr.cbIdt = (uint64_t)u32Val;
4053	}
4054
4055	if (a_fWhat & CPUMCTX_EXTRN_DR7)
4056	{
4057	#ifndef IN_NEM_DARWIN
4058	if (!pVCpu->hmr0.s.fUsingHyperDR7)
4059	#endif
4060	{
4061	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pVCpu->cpum.GstCtx.dr[7]);
4062	AssertRC(rc);
4063	}
4064	}
4065
4066	if (a_fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
4067	{
4068	int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pVCpu->cpum.GstCtx.SysEnter.eip); AssertRC(rc1);
4069	int const rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pVCpu->cpum.GstCtx.SysEnter.esp); AssertRC(rc2);
4070	uint32_t u32Val;
4071	int const rc3 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc3);
4072	pVCpu->cpum.GstCtx.SysEnter.cs = u32Val;
4073	}
4074
4075	#ifndef IN_NEM_DARWIN
4076	if (a_fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
4077	{
4078	if ( (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
4079	&& pVM->hmr0.s.fAllow64BitGuests)
4080	pVCpu->cpum.GstCtx.msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
4081	}
4082
4083	if (a_fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
4084	{
4085	if ( (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
4086	&& pVM->hmr0.s.fAllow64BitGuests)
4087	{
4088	pVCpu->cpum.GstCtx.msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
4089	pVCpu->cpum.GstCtx.msrSTAR = ASMRdMsr(MSR_K6_STAR);
4090	pVCpu->cpum.GstCtx.msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
4091	}
4092	}
4093
4094	if (a_fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
4095	{
4096	int const rc1 = vmxHCImportGuestTscAuxAndOtherMsrs(pVCpu, pVmcsInfo, fEFlags);
4097	AssertRCReturn(rc1, rc1);
4098	}
4099	#else
4100	NOREF(pVM);
4101	#endif
4102
4103	if (a_fWhat & CPUMCTX_EXTRN_CR0)
4104	vmxHCImportGuestCr0(pVCpu, pVmcsInfo);
4105
4106	if (a_fWhat & CPUMCTX_EXTRN_CR4)
4107	vmxHCImportGuestCr4(pVCpu, pVmcsInfo);
4108
4109	if (a_fWhat & CPUMCTX_EXTRN_CR3)
4110	vmxHCImportGuestCr3(pVCpu);
4111
4112	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4113	if (a_fWhat & CPUMCTX_EXTRN_HWVIRT)
4114	{
4115	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
4116	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
4117	{
4118	Assert(CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx));
4119	int const rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
4120	AssertRCReturn(rc, rc);
4121	}
4122	}
4123	#endif
4124
4125	/* Update fExtrn. */
4126	pVCpu->cpum.GstCtx.fExtrn &= ~a_fWhat;
4127
4128	/* If everything has been imported, clear the HM keeper bit. */
4129	if (!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
4130	{
4131	#ifndef IN_NEM_DARWIN
4132	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
4133	#else
4134	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
4135	#endif
4136	Assert(!pVCpu->cpum.GstCtx.fExtrn);
4137	}
4138
4139	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
4140
4141	/*
4142	* Honor any pending CR3 updates.
4143	*
4144	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
4145	* -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
4146	* -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
4147	*
4148	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
4149	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
4150	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
4151	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
4152	*
4153	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
4154	*
4155	* The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
4156	*/
4157	#ifndef IN_NEM_DARWIN
4158	if (!(a_fWhat & CPUMCTX_EXTRN_CR3)
4159	? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) \|\| !VMMRZCallRing3IsEnabled(pVCpu))
4160	: !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) \|\| !VMMRZCallRing3IsEnabled(pVCpu) )
4161	return VINF_SUCCESS;
4162	ASMSetFlags(fEFlags);
4163	#else
4164	if (!(a_fWhat & CPUMCTX_EXTRN_CR3)
4165	? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
4166	: !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) )
4167	return VINF_SUCCESS;
4168	RT_NOREF_PV(fEFlags);
4169	#endif
4170
4171	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
4172	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
4173	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
4174	return VINF_SUCCESS;
4175	}
4176
4177
4178	/**
4179	* Internal state fetcher.
4180	*
4181	* @returns VBox status code.
4182	* @param pVCpu The cross context virtual CPU structure.
4183	* @param pVmcsInfo The VMCS info. object.
4184	* @param pszCaller For logging.
4185	* @tparam a_fWhat What needs to be imported, CPUMCTX_EXTRN_XXX.
4186	* @tparam a_fDoneLocal What's ASSUMED to have been retrieved locally
4187	* already. This is ORed together with @a a_fWhat when
4188	* calculating what needs fetching (just for safety).
4189	* @tparam a_fDonePostExit What's ASSUMED to been been retrieved by
4190	* hmR0VmxPostRunGuest()/nemR3DarwinHandleExitCommon()
4191	* already. This is ORed together with @a a_fWhat when
4192	* calculating what needs fetching (just for safety).
4193	*/
4194	template<uint64_t const a_fWhat,
4195	uint64_t const a_fDoneLocal = 0,
4196	uint64_t const a_fDonePostExit = 0
4197	#ifndef IN_NEM_DARWIN
4198	\| CPUMCTX_EXTRN_INHIBIT_INT
4199	\| CPUMCTX_EXTRN_INHIBIT_NMI
4200	# if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
4201	\| HMVMX_CPUMCTX_EXTRN_ALL
4202	# elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
4203	\| CPUMCTX_EXTRN_RFLAGS
4204	# endif
4205	#else /* IN_NEM_DARWIN */
4206	\| CPUMCTX_EXTRN_ALL /** @todo optimize */
4207	#endif /* IN_NEM_DARWIN */
4208	>
4209	DECLINLINE(int) vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, const char *pszCaller)
4210	{
4211	RT_NOREF_PV(pszCaller);
4212	if ((a_fWhat \| a_fDoneLocal \| a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL)
4213	{
4214	#ifndef IN_NEM_DARWIN
4215	/*
4216	* We disable interrupts to make the updating of the state and in particular
4217	* the fExtrn modification atomic wrt to preemption hooks.
4218	*/
4219	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
4220	#else
4221	RTCCUINTREG const fEFlags = 0;
4222	#endif
4223
4224	/*
4225	* We combine all three parameters and take the (probably) inlined optimized
4226	* code path for the new things specified in a_fWhat.
4227	*
4228	* As a tweak to deal with exits that have INHIBIT_INT/NMI active, causing
4229	* vmxHCImportGuestIntrState to automatically fetch both RIP & RFLAGS, we
4230	* also take the streamlined path when both of these are cleared in fExtrn
4231	* already. vmxHCImportGuestStateInner checks fExtrn before fetching. This
4232	* helps with MWAIT and HLT exits that always inhibit IRQs on many platforms.
4233	*/
4234	uint64_t const fWhatToDo = pVCpu->cpum.GstCtx.fExtrn
4235	& ((a_fWhat \| a_fDoneLocal \| a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL);
4236	if (RT_LIKELY( ( fWhatToDo == (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal \| a_fDonePostExit))
4237	\|\| fWhatToDo == ( a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal \| a_fDonePostExit)
4238	& ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)) /* fetch with INHIBIT_INT/NMI */))
4239	&& (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal \| a_fDonePostExit)) != 0 /* just in case */)
4240	{
4241	int const rc = vmxHCImportGuestStateInner< a_fWhat
4242	& HMVMX_CPUMCTX_EXTRN_ALL
4243	& ~(a_fDoneLocal \| a_fDonePostExit)>(pVCpu, pVmcsInfo, fEFlags);
4244	#ifndef IN_NEM_DARWIN
4245	ASMSetFlags(fEFlags);
4246	#endif
4247	return rc;
4248	}
4249
4250	#ifndef IN_NEM_DARWIN
4251	ASMSetFlags(fEFlags);
4252	#endif
4253
4254	/*
4255	* We shouldn't normally get here, but it may happen when executing
4256	* in the debug run-loops. Typically, everything should already have
4257	* been fetched then. Otherwise call the fallback state import function.
4258	*/
4259	if (fWhatToDo == 0)
4260	{ /* hope the cause was the debug loop or something similar */ }
4261	else
4262	{
4263	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestStateFallback);
4264	Log11Func(("a_fWhat=%#RX64/%#RX64/%#RX64 fExtrn=%#RX64 => %#RX64 - Taking inefficient code path from %s!\n",
4265	a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL, a_fDoneLocal & HMVMX_CPUMCTX_EXTRN_ALL,
4266	a_fDonePostExit & HMVMX_CPUMCTX_EXTRN_ALL, pVCpu->cpum.GstCtx.fExtrn, fWhatToDo, pszCaller));
4267	return vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, a_fWhat \| a_fDoneLocal \| a_fDonePostExit);
4268	}
4269	}
4270	return VINF_SUCCESS;
4271	}
4272
4273
4274	/**
4275	* Check per-VM and per-VCPU force flag actions that require us to go back to
4276	* ring-3 for one reason or another.
4277	*
4278	* @returns Strict VBox status code (i.e. informational status codes too)
4279	* @retval VINF_SUCCESS if we don't have any actions that require going back to
4280	* ring-3.
4281	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
4282	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
4283	* interrupts)
4284	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
4285	* all EMTs to be in ring-3.
4286	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
4287	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
4288	* to the EM loop.
4289	*
4290	* @param pVCpu The cross context virtual CPU structure.
4291	* @param fIsNestedGuest Flag whether this is for a for a pending nested guest event.
4292	* @param fStepping Whether we are single-stepping the guest using the
4293	* hypervisor debugger.
4294	*
4295	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
4296	* is no longer in VMX non-root mode.
4297	*/
4298	static VBOXSTRICTRC vmxHCCheckForceFlags(PVMCPUCC pVCpu, bool fIsNestedGuest, bool fStepping)
4299	{
4300	#ifndef IN_NEM_DARWIN
4301	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4302	#endif
4303
4304	/*
4305	* Update pending interrupts into the APIC's IRR.
4306	*/
4307	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
4308	APICUpdatePendingInterrupts(pVCpu);
4309
4310	/*
4311	* Anything pending? Should be more likely than not if we're doing a good job.
4312	*/
4313	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4314	if ( !fStepping
4315	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
4316	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
4317	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
4318	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
4319	return VINF_SUCCESS;
4320
4321	/* Pending PGM C3 sync. */
4322	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
4323	{
4324	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4325	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
4326	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
4327	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4328	if (rcStrict != VINF_SUCCESS)
4329	{
4330	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
4331	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
4332	return rcStrict;
4333	}
4334	}
4335
4336	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
4337	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
4338	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4339	{
4340	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHmToR3FF);
4341	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
4342	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d (fVM=%#RX64 fCpu=%#RX64)\n",
4343	rc, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
4344	return rc;
4345	}
4346
4347	/* Pending VM request packets, such as hardware interrupts. */
4348	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
4349	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
4350	{
4351	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchVmReq);
4352	Log4Func(("Pending VM request forcing us back to ring-3\n"));
4353	return VINF_EM_PENDING_REQUEST;
4354	}
4355
4356	/* Pending PGM pool flushes. */
4357	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4358	{
4359	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchPgmPoolFlush);
4360	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4361	return VINF_PGM_POOL_FLUSH_PENDING;
4362	}
4363
4364	/* Pending DMA requests. */
4365	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4366	{
4367	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchDma);
4368	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4369	return VINF_EM_RAW_TO_R3;
4370	}
4371
4372	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4373	/*
4374	* Pending nested-guest events.
4375	*
4376	* Please note the priority of these events are specified and important.
4377	* See Intel spec. 29.4.3.2 "APIC-Write Emulation".
4378	* See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
4379	*
4380	* Interrupt-window and NMI-window VM-exits for the nested-guest need not be
4381	* handled here. They'll be handled by the hardware while executing the nested-guest
4382	* or by us when we injecting events that are not part of VM-entry of the nested-guest.
4383	*/
4384	if (fIsNestedGuest)
4385	{
4386	/* Pending nested-guest APIC-write (may or may not cause a VM-exit). */
4387	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
4388	{
4389	Log4Func(("Pending nested-guest APIC-write\n"));
4390	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
4391	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
4392	if ( rcStrict == VINF_SUCCESS
4393	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
4394	return rcStrict;
4395	}
4396
4397	/* Pending nested-guest monitor-trap flag (MTF). */
4398	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
4399	{
4400	Log4Func(("Pending nested-guest MTF\n"));
4401	VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
4402	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
4403	return rcStrict;
4404	}
4405
4406	/* Pending nested-guest VMX-preemption timer expired. */
4407	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
4408	{
4409	Log4Func(("Pending nested-guest preempt timer\n"));
4410	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
4411	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
4412	return rcStrict;
4413	}
4414	}
4415	#else
4416	NOREF(fIsNestedGuest);
4417	#endif
4418
4419	return VINF_SUCCESS;
4420	}
4421
4422
4423	/**
4424	* Converts any TRPM trap into a pending HM event. This is typically used when
4425	* entering from ring-3 (not longjmp returns).
4426	*
4427	* @param pVCpu The cross context virtual CPU structure.
4428	*/
4429	static void vmxHCTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
4430	{
4431	Assert(TRPMHasTrap(pVCpu));
4432	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
4433
4434	uint8_t uVector;
4435	TRPMEVENT enmTrpmEvent;
4436	uint32_t uErrCode;
4437	RTGCUINTPTR GCPtrFaultAddress;
4438	uint8_t cbInstr;
4439	bool fIcebp;
4440
4441	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
4442	AssertRC(rc);
4443
4444	uint32_t u32IntInfo;
4445	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
4446	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
4447
4448	rc = TRPMResetTrap(pVCpu);
4449	AssertRC(rc);
4450	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
4451	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
4452
4453	vmxHCSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
4454	}
4455
4456
4457	/**
4458	* Converts the pending HM event into a TRPM trap.
4459	*
4460	* @param pVCpu The cross context virtual CPU structure.
4461	*/
4462	static void vmxHCPendingEventToTrpmTrap(PVMCPUCC pVCpu)
4463	{
4464	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
4465
4466	/* If a trap was already pending, we did something wrong! */
4467	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
4468
4469	uint32_t const u32IntInfo = VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo;
4470	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
4471	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
4472
4473	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
4474
4475	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
4476	AssertRC(rc);
4477
4478	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
4479	TRPMSetErrorCode(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode);
4480
4481	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
4482	TRPMSetFaultAddress(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress);
4483	else
4484	{
4485	uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
4486	switch (uVectorType)
4487	{
4488	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
4489	TRPMSetTrapDueToIcebp(pVCpu);
4490	RT_FALL_THRU();
4491	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
4492	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
4493	{
4494	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4495	\|\| ( uVector == X86_XCPT_BP /* INT3 */
4496	\|\| uVector == X86_XCPT_OF /* INTO */
4497	\|\| uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
4498	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
4499	TRPMSetInstrLength(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.cbInstr);
4500	break;
4501	}
4502	}
4503	}
4504
4505	/* We're now done converting the pending event. */
4506	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
4507	}
4508
4509
4510	/**
4511	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
4512	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
4513	*
4514	* @param pVCpu The cross context virtual CPU structure.
4515	* @param pVmcsInfo The VMCS info. object.
4516	*/
4517	static void vmxHCSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4518	{
4519	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
4520	{
4521	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
4522	{
4523	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
4524	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4525	AssertRC(rc);
4526	}
4527	Log4Func(("Enabled interrupt-window exiting\n"));
4528	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
4529	}
4530
4531
4532	/**
4533	* Clears the interrupt-window exiting control in the VMCS.
4534	*
4535	* @param pVCpu The cross context virtual CPU structure.
4536	* @param pVmcsInfo The VMCS info. object.
4537	*/
4538	DECLINLINE(void) vmxHCClearIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4539	{
4540	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
4541	{
4542	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
4543	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4544	AssertRC(rc);
4545	Log4Func(("Disabled interrupt-window exiting\n"));
4546	}
4547	}
4548
4549
4550	/**
4551	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
4552	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
4553	*
4554	* @param pVCpu The cross context virtual CPU structure.
4555	* @param pVmcsInfo The VMCS info. object.
4556	*/
4557	static void vmxHCSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4558	{
4559	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
4560	{
4561	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
4562	{
4563	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
4564	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4565	AssertRC(rc);
4566	Log4Func(("Enabled NMI-window exiting\n"));
4567	}
4568	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
4569	}
4570
4571
4572	/**
4573	* Clears the NMI-window exiting control in the VMCS.
4574	*
4575	* @param pVCpu The cross context virtual CPU structure.
4576	* @param pVmcsInfo The VMCS info. object.
4577	*/
4578	DECLINLINE(void) vmxHCClearNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4579	{
4580	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
4581	{
4582	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
4583	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4584	AssertRC(rc);
4585	Log4Func(("Disabled NMI-window exiting\n"));
4586	}
4587	}
4588
4589
4590	/**
4591	* Injects an event into the guest upon VM-entry by updating the relevant fields
4592	* in the VM-entry area in the VMCS.
4593	*
4594	* @returns Strict VBox status code (i.e. informational status codes too).
4595	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
4596	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
4597	*
4598	* @param pVCpu The cross context virtual CPU structure.
4599	* @param pVmcsInfo The VMCS info object.
4600	* @param fIsNestedGuest Flag whether this is for a for a pending nested guest event.
4601	* @param pEvent The event being injected.
4602	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
4603	* will be updated if necessary. This cannot not be NULL.
4604	* @param fStepping Whether we're single-stepping guest execution and should
4605	* return VINF_EM_DBG_STEPPED if the event is injected
4606	* directly (registers modified by us, not by hardware on
4607	* VM-entry).
4608	*/
4609	static VBOXSTRICTRC vmxHCInjectEventVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, PCHMEVENT pEvent,
4610	bool fStepping, uint32_t *pfIntrState)
4611	{
4612	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
4613	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
4614	Assert(pfIntrState);
4615
4616	#ifdef IN_NEM_DARWIN
4617	RT_NOREF(fIsNestedGuest, fStepping, pfIntrState);
4618	#endif
4619
4620	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4621	uint32_t u32IntInfo = pEvent->u64IntInfo;
4622	uint32_t const u32ErrCode = pEvent->u32ErrCode;
4623	uint32_t const cbInstr = pEvent->cbInstr;
4624	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
4625	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
4626	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
4627
4628	#ifdef VBOX_STRICT
4629	/*
4630	* Validate the error-code-valid bit for hardware exceptions.
4631	* No error codes for exceptions in real-mode.
4632	*
4633	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
4634	*/
4635	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
4636	&& !CPUMIsGuestInRealModeEx(pCtx))
4637	{
4638	switch (uVector)
4639	{
4640	case X86_XCPT_PF:
4641	case X86_XCPT_DF:
4642	case X86_XCPT_TS:
4643	case X86_XCPT_NP:
4644	case X86_XCPT_SS:
4645	case X86_XCPT_GP:
4646	case X86_XCPT_AC:
4647	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
4648	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
4649	RT_FALL_THRU();
4650	default:
4651	break;
4652	}
4653	}
4654
4655	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
4656	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
4657	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
4658	#endif
4659
4660	RT_NOREF(uVector);
4661	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
4662	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
4663	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
4664	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
4665	{
4666	Assert(uVector <= X86_XCPT_LAST);
4667	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI \|\| uVector == X86_XCPT_NMI);
4668	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT \|\| uVector == X86_XCPT_DB);
4669	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedXcpts[uVector]);
4670	}
4671	else
4672	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedIrqs[uVector & MASK_INJECT_IRQ_STAT]);
4673
4674	/*
4675	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
4676	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
4677	* interrupt handler in the (real-mode) guest.
4678	*
4679	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
4680	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
4681	*/
4682	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
4683	{
4684	#ifndef IN_NEM_DARWIN
4685	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest)
4686	#endif
4687	{
4688	/*
4689	* For CPUs with unrestricted guest execution enabled and with the guest
4690	* in real-mode, we must not set the deliver-error-code bit.
4691	*
4692	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
4693	*/
4694	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
4695	}
4696	#ifndef IN_NEM_DARWIN
4697	else
4698	{
4699	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4700	Assert(PDMVmmDevHeapIsEnabled(pVM));
4701	Assert(pVM->hm.s.vmx.pRealModeTSS);
4702	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
4703
4704	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
4705	int rc2 = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
4706	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
4707	AssertRCReturn(rc2, rc2);
4708
4709	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
4710	size_t const cbIdtEntry = sizeof(X86IDTR16);
4711	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
4712	{
4713	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
4714	if (uVector == X86_XCPT_DF)
4715	return VINF_EM_RESET;
4716
4717	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
4718	No error codes for exceptions in real-mode. */
4719	if (uVector == X86_XCPT_GP)
4720	{
4721	static HMEVENT const s_EventXcptDf
4722	= HMEVENT_INIT_ONLY_INT_INFO( RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
4723	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
4724	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4725	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1));
4726	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptDf, fStepping, pfIntrState);
4727	}
4728
4729	/*
4730	* If we're injecting an event with no valid IDT entry, inject a #GP.
4731	* No error codes for exceptions in real-mode.
4732	*
4733	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
4734	*/
4735	static HMEVENT const s_EventXcptGp
4736	= HMEVENT_INIT_ONLY_INT_INFO( RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
4737	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
4738	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4739	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1));
4740	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptGp, fStepping, pfIntrState);
4741	}
4742
4743	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
4744	uint16_t uGuestIp = pCtx->ip;
4745	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
4746	{
4747	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
4748	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
4749	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
4750	}
4751	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
4752	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
4753
4754	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
4755	X86IDTR16 IdtEntry;
4756	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
4757	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
4758	AssertRCReturn(rc2, rc2);
4759
4760	/* Construct the stack frame for the interrupt/exception handler. */
4761	VBOXSTRICTRC rcStrict;
4762	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, (uint16_t)pCtx->eflags.u);
4763	if (rcStrict == VINF_SUCCESS)
4764	{
4765	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
4766	if (rcStrict == VINF_SUCCESS)
4767	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
4768	}
4769
4770	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
4771	if (rcStrict == VINF_SUCCESS)
4772	{
4773	pCtx->eflags.u &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
4774	pCtx->rip = IdtEntry.offSel;
4775	pCtx->cs.Sel = IdtEntry.uSel;
4776	pCtx->cs.ValidSel = IdtEntry.uSel;
4777	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
4778	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
4779	&& uVector == X86_XCPT_PF)
4780	pCtx->cr2 = GCPtrFault;
4781
4782	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
4783	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
4784	\| HM_CHANGED_GUEST_RSP);
4785
4786	/*
4787	* If we delivered a hardware exception (other than an NMI) and if there was
4788	* block-by-STI in effect, we should clear it.
4789	*/
4790	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
4791	{
4792	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
4793	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
4794	Log4Func(("Clearing inhibition due to STI\n"));
4795	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4796	}
4797
4798	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
4799	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
4800
4801	/*
4802	* The event has been truly dispatched to the guest. Mark it as no longer pending so
4803	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
4804	*/
4805	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
4806
4807	/*
4808	* If we eventually support nested-guest execution without unrestricted guest execution,
4809	* we should set fInterceptEvents here.
4810	*/
4811	Assert(!fIsNestedGuest);
4812
4813	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
4814	if (fStepping)
4815	rcStrict = VINF_EM_DBG_STEPPED;
4816	}
4817	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
4818	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
4819	return rcStrict;
4820	}
4821	#else
4822	RT_NOREF(pVmcsInfo);
4823	#endif
4824	}
4825
4826	/*
4827	* Validate.
4828	*/
4829	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
4830	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
4831
4832	/*
4833	* Inject the event into the VMCS.
4834	*/
4835	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
4836	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
4837	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
4838	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
4839	AssertRC(rc);
4840
4841	/*
4842	* Update guest CR2 if this is a page-fault.
4843	*/
4844	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
4845	pCtx->cr2 = GCPtrFault;
4846
4847	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
4848	return VINF_SUCCESS;
4849	}
4850
4851
4852	/**
4853	* Evaluates the event to be delivered to the guest and sets it as the pending
4854	* event.
4855	*
4856	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
4857	* exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
4858	* NOT restore these force-flags.
4859	*
4860	* @returns Strict VBox status code (i.e. informational status codes too).
4861	* @param pVCpu The cross context virtual CPU structure.
4862	* @param pVmcsInfo The VMCS information structure.
4863	* @param pfIntrState Where to store the updated VMX guest-interruptibility
4864	* state.
4865	*/
4866	static VBOXSTRICTRC vmxHCEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t *pfIntrState)
4867	{
4868	Assert(pfIntrState);
4869	Assert(!TRPMHasTrap(pVCpu));
4870
4871	*pfIntrState = vmxHCGetGuestIntrStateWithUpdate(pVCpu);
4872
4873	/*
4874	* Evaluate if a new event needs to be injected.
4875	* An event that's already pending has already performed all necessary checks.
4876	*/
4877	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
4878	&& !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
4879	{
4880	/** @todo SMI. SMIs take priority over NMIs. */
4881
4882	/*
4883	* NMIs.
4884	* NMIs take priority over external interrupts.
4885	*/
4886	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
4887	{
4888	if (!CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
4889	{
4890	/* Finally, inject the NMI and we're done. */
4891	vmxHCSetPendingXcptNmi(pVCpu);
4892	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
4893	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4894	return VINF_SUCCESS;
4895	}
4896	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4897	}
4898	else
4899	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4900
4901	/*
4902	* External interrupts (PIC/APIC).
4903	*/
4904	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
4905	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
4906	{
4907	Assert(!DBGFIsStepping(pVCpu));
4908	int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
4909	AssertRC(rc);
4910
4911	if (pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF)
4912	{
4913	/*
4914	* Once PDMGetInterrupt() returns an interrupt we -must- deliver it.
4915	* We cannot re-request the interrupt from the controller again.
4916	*/
4917	uint8_t u8Interrupt;
4918	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
4919	if (RT_SUCCESS(rc))
4920	vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
4921	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
4922	{
4923	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchTprMaskedIrq);
4924	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4925	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
4926	/*
4927	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
4928	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
4929	* need to re-set this force-flag here.
4930	*/
4931	}
4932	else
4933	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchGuestIrq);
4934
4935	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
4936	return VINF_SUCCESS;
4937	}
4938	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
4939	}
4940	else
4941	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
4942	}
4943	else
4944	{
4945	/*
4946	* An event is being injected or we are in an interrupt shadow.
4947	* If another event is pending currently, instruct VT-x to cause a VM-exit as
4948	* soon as the guest is ready to accept it.
4949	*/
4950	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
4951	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4952	else
4953	{
4954	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT));
4955	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
4956	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
4957	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
4958	else
4959	{
4960	/* It's possible that interrupt-window exiting is still active, clear it as it's now unnecessary. */
4961	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
4962	}
4963	}
4964	}
4965
4966	return VINF_SUCCESS;
4967	}
4968
4969
4970	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4971	/**
4972	* Evaluates the event to be delivered to the nested-guest and sets it as the
4973	* pending event.
4974	*
4975	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
4976	* exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
4977	* NOT restore these force-flags.
4978	*
4979	* @returns Strict VBox status code (i.e. informational status codes too).
4980	* @param pVCpu The cross context virtual CPU structure.
4981	* @param pVmcsInfo The VMCS information structure.
4982	* @param pfIntrState Where to store the updated VMX guest-interruptibility
4983	* state.
4984	*
4985	* @remarks The guest must be in VMX non-root mode.
4986	*/
4987	static VBOXSTRICTRC vmxHCEvaluatePendingEventNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t *pfIntrState)
4988	{
4989	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4990
4991	Assert(pfIntrState);
4992	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
4993	Assert(!TRPMHasTrap(pVCpu));
4994
4995	*pfIntrState = vmxHCGetGuestIntrStateWithUpdate(pVCpu);
4996
4997	/*
4998	* If we are injecting an event, all necessary checks have been performed.
4999	* Any interrupt-window or NMI-window exiting would have been setup by the
5000	* nested-guest while we merged controls.
5001	*/
5002	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
5003	return VINF_SUCCESS;
5004
5005	/*
5006	* An event injected by VMLAUNCH/VMRESUME instruction emulation should've been
5007	* made pending (TRPM to HM event) and would be handled above if we resumed
5008	* execution in HM. If somehow we fell back to emulation after the
5009	* VMLAUNCH/VMRESUME instruction, it would have been handled in iemRaiseXcptOrInt
5010	* (calling iemVmxVmexitEvent). Thus, if we get here the nested-hypervisor's VMX
5011	* intercepts should be active and any events pending here have been generated
5012	* while executing the guest in VMX non-root mode after virtual VM-entry completed.
5013	*/
5014	Assert(CPUMIsGuestVmxInterceptEvents(pCtx));
5015
5016	/*
5017	* Interrupt shadows MAY block NMIs.
5018	* They also blocks external-interrupts and MAY block external-interrupt VM-exits.
5019	*
5020	* See Intel spec. 24.4.2 "Guest Non-Register State".
5021	* See Intel spec. 25.4.1 "Event Blocking".
5022	*/
5023	if (!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
5024	{ /* likely */ }
5025	else
5026	return VINF_SUCCESS;
5027
5028	/** @todo SMI. SMIs take priority over NMIs. */
5029
5030	/*
5031	* NMIs.
5032	* NMIs take priority over interrupts.
5033	*/
5034	if (!CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
5035	{
5036	/*
5037	* Nested-guest NMI-window exiting.
5038	* The NMI-window exit must happen regardless of whether an NMI is pending
5039	* provided virtual-NMI blocking is not in effect.
5040	*
5041	* See Intel spec. 25.2 "Other Causes Of VM Exits".
5042	*/
5043	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
5044	&& !CPUMIsGuestVmxVirtNmiBlocking(pCtx))
5045	{
5046	Assert(CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT));
5047	return IEMExecVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
5048	}
5049
5050	/*
5051	* For a nested-guest, the FF always indicates the outer guest's ability to
5052	* receive an NMI while the guest-interruptibility state bit depends on whether
5053	* the nested-hypervisor is using virtual-NMIs.
5054	*
5055	* It is very important that we also clear the force-flag if we are causing
5056	* an NMI VM-exit as it is the responsibility of the nested-hypervisor to deal
5057	* with re-injecting or discarding the NMI. This fixes the bug that showed up
5058	* with SMP Windows Server 2008 R2 with Hyper-V enabled, see @bugref{10318#c19}.
5059	*/
5060	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5061	{
5062	if (CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
5063	return IEMExecVmxVmexitXcptNmi(pVCpu);
5064	vmxHCSetPendingXcptNmi(pVCpu);
5065	return VINF_SUCCESS;
5066	}
5067	}
5068
5069	/*
5070	* Nested-guest interrupt-window exiting.
5071	*
5072	* We must cause the interrupt-window exit regardless of whether an interrupt is pending
5073	* provided virtual interrupts are enabled.
5074	*
5075	* See Intel spec. 25.2 "Other Causes Of VM Exits".
5076	* See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
5077	*/
5078	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
5079	&& CPUMIsGuestVmxVirtIntrEnabled(pCtx))
5080	{
5081	Assert(CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT));
5082	return IEMExecVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
5083	}
5084
5085	/*
5086	* External interrupts (PIC/APIC).
5087	*
5088	* When "External interrupt exiting" is set the VM-exit happens regardless of RFLAGS.IF.
5089	* When it isn't set, RFLAGS.IF controls delivery of the interrupt as always.
5090	* This fixes a nasty SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued
5091	* by other VM-exits (like a preemption timer), see @bugref{9562#c18}.
5092	*
5093	* NMIs block external interrupts as they are dispatched through the interrupt gate (vector 2)
5094	* which automatically clears EFLAGS.IF. Also it's possible an NMI handler could enable interrupts
5095	* and thus we should not check for NMI inhibition here.
5096	*
5097	* See Intel spec. 25.4.1 "Event Blocking".
5098	* See Intel spec. 6.8.1 "Masking Maskable Hardware Interrupts".
5099	*/
5100	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
5101	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
5102	{
5103	Assert(!DBGFIsStepping(pVCpu));
5104	int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
5105	AssertRC(rc);
5106	if (CPUMIsGuestVmxPhysIntrEnabled(pCtx))
5107	{
5108	/* Nested-guest external interrupt VM-exit. */
5109	if ( CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
5110	&& !CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
5111	{
5112	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
5113	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5114	return rcStrict;
5115	}
5116
5117	/*
5118	* Fetch the external interrupt from the interrupt controller.
5119	* Once PDMGetInterrupt() returns an interrupt we -must- deliver it or pass it to
5120	* the nested-hypervisor. We cannot re-request the interrupt from the controller again.
5121	*/
5122	uint8_t u8Interrupt;
5123	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5124	if (RT_SUCCESS(rc))
5125	{
5126	/* Nested-guest external interrupt VM-exit when the "acknowledge interrupt on exit" is enabled. */
5127	if (CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
5128	{
5129	Assert(CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_ACK_EXT_INT));
5130	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
5131	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5132	return rcStrict;
5133	}
5134	vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
5135	return VINF_SUCCESS;
5136	}
5137	}
5138	}
5139	return VINF_SUCCESS;
5140	}
5141	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
5142
5143
5144	/**
5145	* Injects any pending events into the guest if the guest is in a state to
5146	* receive them.
5147	*
5148	* @returns Strict VBox status code (i.e. informational status codes too).
5149	* @param pVCpu The cross context virtual CPU structure.
5150	* @param pVmcsInfo The VMCS information structure.
5151	* @param fIsNestedGuest Flag whether the event injection happens for a nested guest.
5152	* @param fIntrState The VT-x guest-interruptibility state.
5153	* @param fStepping Whether we are single-stepping the guest using the
5154	* hypervisor debugger and should return
5155	* VINF_EM_DBG_STEPPED if the event was dispatched
5156	* directly.
5157	*/
5158	static VBOXSTRICTRC vmxHCInjectPendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest,
5159	uint32_t fIntrState, bool fStepping)
5160	{
5161	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
5162	#ifndef IN_NEM_DARWIN
5163	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5164	#endif
5165
5166	#ifdef VBOX_STRICT
5167	/*
5168	* Verify guest-interruptibility state.
5169	*
5170	* We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
5171	* since injecting an event may modify the interruptibility state and we must thus always
5172	* use fIntrState.
5173	*/
5174	{
5175	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5176	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
5177	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
5178	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5179	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
5180	Assert(!TRPMHasTrap(pVCpu));
5181	NOREF(fBlockMovSS); NOREF(fBlockSti);
5182	}
5183	#endif
5184
5185	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5186	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
5187	{
5188	/*
5189	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
5190	* pending even while injecting an event and in this case, we want a VM-exit as soon as
5191	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
5192	*
5193	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
5194	*/
5195	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo);
5196	#ifdef VBOX_STRICT
5197	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
5198	{
5199	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
5200	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
5201	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
5202	}
5203	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
5204	{
5205	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
5206	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
5207	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
5208	}
5209	#endif
5210	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
5211	uIntType));
5212
5213	/*
5214	* Inject the event and get any changes to the guest-interruptibility state.
5215	*
5216	* The guest-interruptibility state may need to be updated if we inject the event
5217	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
5218	*/
5219	rcStrict = vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &VCPU_2_VMXSTATE(pVCpu).Event, fStepping, &fIntrState);
5220	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
5221
5222	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
5223	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterrupt);
5224	else
5225	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectXcpt);
5226	}
5227
5228	/*
5229	* Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
5230	* is an interrupt shadow (block-by-STI or block-by-MOV SS).
5231	*/
5232	if ( (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5233	&& !fIsNestedGuest)
5234	{
5235	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5236
5237	if (!VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
5238	{
5239	/*
5240	* Set or clear the BS bit depending on whether the trap flag is active or not. We need
5241	* to do both since we clear the BS bit from the VMCS while exiting to ring-3.
5242	*/
5243	Assert(!DBGFIsStepping(pVCpu));
5244	uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_TF);
5245	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
5246	fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
5247	AssertRC(rc);
5248	}
5249	else
5250	{
5251	/*
5252	* We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
5253	* hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
5254	* we take care of this case in vmxHCExportSharedDebugState and also the case if
5255	* we use MTF, so just make sure it's called before executing guest-code.
5256	*/
5257	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
5258	}
5259	}
5260	/* else: for nested-guest currently handling while merging controls. */
5261
5262	/*
5263	* Finally, update the guest-interruptibility state.
5264	*
5265	* This is required for the real-on-v86 software interrupt injection, for
5266	* pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
5267	*/
5268	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5269	AssertRC(rc);
5270
5271	/*
5272	* There's no need to clear the VM-entry interruption-information field here if we're not
5273	* injecting anything. VT-x clears the valid bit on every VM-exit.
5274	*
5275	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5276	*/
5277
5278	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
5279	return rcStrict;
5280	}
5281
5282
5283	/**
5284	* Tries to determine what part of the guest-state VT-x has deemed as invalid
5285	* and update error record fields accordingly.
5286	*
5287	* @returns VMX_IGS_* error codes.
5288	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
5289	* wrong with the guest state.
5290	*
5291	* @param pVCpu The cross context virtual CPU structure.
5292	* @param pVmcsInfo The VMCS info. object.
5293	*
5294	* @remarks This function assumes our cache of the VMCS controls
5295	* are valid, i.e. vmxHCCheckCachedVmcsCtls() succeeded.
5296	*/
5297	static uint32_t vmxHCCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
5298	{
5299	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
5300	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { uError = (err); break; } else do { } while (0)
5301
5302	PCPUMCTX pCtx =