VMXAllTemplate.cpp.h

Last change on this file was 108996, checked in by vboxsync, 3 weeks ago
VMM: bugref:3409 Build fix.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 530.0 KB

Line
1	/* $Id: VMXAllTemplate.cpp.h 108996 2025-04-16 06:47:47Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Code template for our own hypervisor and the NEM darwin backend using Apple's Hypervisor.framework.
4	*/
5
6	/*
7	* Copyright (C) 2012-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Defined Constants And Macros *
31	*********************************************************************************************************************************/
32	#if !defined(VMX_VMCS_WRITE_16) \|\| !defined(VMX_VMCS_WRITE_32) \|\| !defined(VMX_VMCS_WRITE_64) \|\| !defined(VMX_VMCS_WRITE_64)
33	# error "At least one of the VMX_VMCS_WRITE_16, VMX_VMCS_WRITE_32, VMX_VMCS_WRITE_64 or VMX_VMCS_WRITE_64 is missing"
34	#endif
35
36
37	#if !defined(VMX_VMCS_READ_16) \|\| !defined(VMX_VMCS_READ_32) \|\| !defined(VMX_VMCS_READ_64) \|\| !defined(VMX_VMCS_READ_64)
38	# error "At least one of the VMX_VMCS_READ_16, VMX_VMCS_READ_32, VMX_VMCS_READ_64 or VMX_VMCS_READ_64 is missing"
39	#endif
40
41	/** Enables condensing of VMREAD instructions, see vmxHCReadToTransient(). */
42	#define HMVMX_WITH_CONDENSED_VMREADS
43
44	/** Use the function table. */
45	#define HMVMX_USE_FUNCTION_TABLE
46
47	/** Determine which tagged-TLB flush handler to use. */
48	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
49	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
50	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
51	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
52
53	/** Assert that all the given fields have been read from the VMCS. */
54	#ifdef VBOX_STRICT
55	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
56	do { \
57	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
58	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
59	} while (0)
60	#else
61	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
62	#endif
63
64	/**
65	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
66	* guest using hardware-assisted VMX.
67	*
68	* This excludes state like GPRs (other than RSP) which are always are
69	* swapped and restored across the world-switch and also registers like EFER,
70	* MSR which cannot be modified by the guest without causing a VM-exit.
71	*/
72	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
73	\| CPUMCTX_EXTRN_RFLAGS \
74	\| CPUMCTX_EXTRN_RSP \
75	\| CPUMCTX_EXTRN_SREG_MASK \
76	\| CPUMCTX_EXTRN_TABLE_MASK \
77	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
78	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
79	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
80	\| CPUMCTX_EXTRN_TSC_AUX \
81	\| CPUMCTX_EXTRN_OTHER_MSRS \
82	\| CPUMCTX_EXTRN_CR0 \
83	\| CPUMCTX_EXTRN_CR3 \
84	\| CPUMCTX_EXTRN_CR4 \
85	\| CPUMCTX_EXTRN_DR7 \
86	\| CPUMCTX_EXTRN_HWVIRT \
87	\| CPUMCTX_EXTRN_INHIBIT_INT \
88	\| CPUMCTX_EXTRN_INHIBIT_NMI)
89
90	/**
91	* Guest-CPU state required for split-lock \#AC handling VM-exits.
92	*/
93	#define HMVMX_CPUMCTX_XPCT_AC ( CPUMCTX_EXTRN_CR0 \
94	\| CPUMCTX_EXTRN_RFLAGS \
95	\| CPUMCTX_EXTRN_SS \
96	\| CPUMCTX_EXTRN_CS)
97
98	/**
99	* Exception bitmap mask for real-mode guests (real-on-v86).
100	*
101	* We need to intercept all exceptions manually except:
102	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
103	* due to bugs in Intel CPUs.
104	* - \#PF need not be intercepted even in real-mode if we have nested paging
105	* support.
106	*/
107	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
108	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
109	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
110	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
111	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
112	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
113	\| RT_BIT(X86_XCPT_XF))
114
115	/** Maximum VM-instruction error number. */
116	#define HMVMX_INSTR_ERROR_MAX 28
117
118	/** Profiling macro. */
119	#ifdef HM_PROFILE_EXIT_DISPATCH
120	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
121	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
122	#else
123	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
124	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
125	#endif
126
127	#ifndef IN_NEM_DARWIN
128	/** Assert that preemption is disabled or covered by thread-context hooks. */
129	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
130	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
131
132	/** Assert that we haven't migrated CPUs when thread-context hooks are not
133	* used. */
134	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
135	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
136	("Illegal migration! Entered on CPU %u Current %u\n", \
137	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()))
138	#else
139	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) do { } while (0)
140	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) do { } while (0)
141	#endif
142
143	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
144	* context. */
145	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
146	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
147	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
148
149	/** Log the VM-exit reason with an easily visible marker to identify it in a
150	* potential sea of logging data. */
151	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
152	do { \
153	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
154	HMGetVmxExitName(a_uExitReason))); \
155	} while (0) \
156
157
158	/*********************************************************************************************************************************
159	* Structures and Typedefs *
160	*********************************************************************************************************************************/
161	/**
162	* Memory operand read or write access.
163	*/
164	typedef enum VMXMEMACCESS
165	{
166	VMXMEMACCESS_READ = 0,
167	VMXMEMACCESS_WRITE = 1
168	} VMXMEMACCESS;
169
170
171	/**
172	* VMX VM-exit handler.
173	*
174	* @returns Strict VBox status code (i.e. informational status codes too).
175	* @param pVCpu The cross context virtual CPU structure.
176	* @param pVmxTransient The VMX-transient structure.
177	*/
178	#ifndef HMVMX_USE_FUNCTION_TABLE
179	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
180	#else
181	typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
182	/** Pointer to VM-exit handler. */
183	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
184	#endif
185
186	/**
187	* VMX VM-exit handler, non-strict status code.
188	*
189	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
190	*
191	* @returns VBox status code, no informational status code returned.
192	* @param pVCpu The cross context virtual CPU structure.
193	* @param pVmxTransient The VMX-transient structure.
194	*
195	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
196	* use of that status code will be replaced with VINF_EM_SOMETHING
197	* later when switching over to IEM.
198	*/
199	#ifndef HMVMX_USE_FUNCTION_TABLE
200	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
201	#else
202	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
203	#endif
204
205
206	/*********************************************************************************************************************************
207	* Internal Functions *
208	*********************************************************************************************************************************/
209	#ifndef HMVMX_USE_FUNCTION_TABLE
210	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
211	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
212	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
213	#else
214	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
215	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
216	#endif
217	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
218	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
219	#endif
220
221	static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
222
223	/** @name VM-exit handler prototypes.
224	* @{
225	*/
226	static FNVMXEXITHANDLER vmxHCExitXcptOrNmi;
227	static FNVMXEXITHANDLER vmxHCExitExtInt;
228	static FNVMXEXITHANDLER vmxHCExitTripleFault;
229	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindow;
230	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindow;
231	static FNVMXEXITHANDLER vmxHCExitTaskSwitch;
232	static FNVMXEXITHANDLER vmxHCExitCpuid;
233	static FNVMXEXITHANDLER vmxHCExitGetsec;
234	static FNVMXEXITHANDLER vmxHCExitHlt;
235	static FNVMXEXITHANDLERNSRC vmxHCExitInvd;
236	static FNVMXEXITHANDLER vmxHCExitInvlpg;
237	static FNVMXEXITHANDLER vmxHCExitRdpmc;
238	static FNVMXEXITHANDLER vmxHCExitVmcall;
239	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
240	static FNVMXEXITHANDLER vmxHCExitVmclear;
241	static FNVMXEXITHANDLER vmxHCExitVmlaunch;
242	static FNVMXEXITHANDLER vmxHCExitVmptrld;
243	static FNVMXEXITHANDLER vmxHCExitVmptrst;
244	static FNVMXEXITHANDLER vmxHCExitVmread;
245	static FNVMXEXITHANDLER vmxHCExitVmresume;
246	static FNVMXEXITHANDLER vmxHCExitVmwrite;
247	static FNVMXEXITHANDLER vmxHCExitVmxoff;
248	static FNVMXEXITHANDLER vmxHCExitVmxon;
249	static FNVMXEXITHANDLER vmxHCExitInvvpid;
250	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
251	static FNVMXEXITHANDLER vmxHCExitInvept;
252	# endif
253	#endif
254	static FNVMXEXITHANDLER vmxHCExitRdtsc;
255	static FNVMXEXITHANDLER vmxHCExitMovCRx;
256	static FNVMXEXITHANDLER vmxHCExitMovDRx;
257	static FNVMXEXITHANDLER vmxHCExitIoInstr;
258	static FNVMXEXITHANDLER vmxHCExitRdmsr;
259	static FNVMXEXITHANDLER vmxHCExitWrmsr;
260	static FNVMXEXITHANDLER vmxHCExitMwait;
261	static FNVMXEXITHANDLER vmxHCExitMtf;
262	static FNVMXEXITHANDLER vmxHCExitMonitor;
263	static FNVMXEXITHANDLER vmxHCExitPause;
264	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThreshold;
265	static FNVMXEXITHANDLER vmxHCExitApicAccess;
266	static FNVMXEXITHANDLER vmxHCExitEptViolation;
267	static FNVMXEXITHANDLER vmxHCExitEptMisconfig;
268	static FNVMXEXITHANDLER vmxHCExitRdtscp;
269	static FNVMXEXITHANDLER vmxHCExitPreemptTimer;
270	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvd;
271	static FNVMXEXITHANDLER vmxHCExitXsetbv;
272	static FNVMXEXITHANDLER vmxHCExitInvpcid;
273	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
274	static FNVMXEXITHANDLERNSRC vmxHCExitSetPendingXcptUD;
275	#endif
276	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestState;
277	static FNVMXEXITHANDLERNSRC vmxHCExitErrUnexpected;
278	/** @} */
279
280	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
281	/** @name Nested-guest VM-exit handler prototypes.
282	* @{
283	*/
284	static FNVMXEXITHANDLER vmxHCExitXcptOrNmiNested;
285	static FNVMXEXITHANDLER vmxHCExitTripleFaultNested;
286	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindowNested;
287	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindowNested;
288	static FNVMXEXITHANDLER vmxHCExitTaskSwitchNested;
289	static FNVMXEXITHANDLER vmxHCExitHltNested;
290	static FNVMXEXITHANDLER vmxHCExitInvlpgNested;
291	static FNVMXEXITHANDLER vmxHCExitRdpmcNested;
292	static FNVMXEXITHANDLER vmxHCExitVmreadVmwriteNested;
293	static FNVMXEXITHANDLER vmxHCExitRdtscNested;
294	static FNVMXEXITHANDLER vmxHCExitMovCRxNested;
295	static FNVMXEXITHANDLER vmxHCExitMovDRxNested;
296	static FNVMXEXITHANDLER vmxHCExitIoInstrNested;
297	static FNVMXEXITHANDLER vmxHCExitRdmsrNested;
298	static FNVMXEXITHANDLER vmxHCExitWrmsrNested;
299	static FNVMXEXITHANDLER vmxHCExitMwaitNested;
300	static FNVMXEXITHANDLER vmxHCExitMtfNested;
301	static FNVMXEXITHANDLER vmxHCExitMonitorNested;
302	static FNVMXEXITHANDLER vmxHCExitPauseNested;
303	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThresholdNested;
304	static FNVMXEXITHANDLER vmxHCExitApicAccessNested;
305	static FNVMXEXITHANDLER vmxHCExitApicWriteNested;
306	static FNVMXEXITHANDLER vmxHCExitVirtEoiNested;
307	static FNVMXEXITHANDLER vmxHCExitRdtscpNested;
308	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvdNested;
309	static FNVMXEXITHANDLER vmxHCExitInvpcidNested;
310	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestStateNested;
311	static FNVMXEXITHANDLER vmxHCExitInstrNested;
312	static FNVMXEXITHANDLER vmxHCExitInstrWithInfoNested;
313	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
314	static FNVMXEXITHANDLER vmxHCExitEptViolationNested;
315	static FNVMXEXITHANDLER vmxHCExitEptMisconfigNested;
316	# endif
317	/** @} */
318	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
319
320
321	/*********************************************************************************************************************************
322	* Global Variables *
323	*********************************************************************************************************************************/
324	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
325	/**
326	* Array of all VMCS fields.
327	* Any fields added to the VT-x spec. should be added here.
328	*
329	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
330	* of nested-guests.
331	*/
332	static const uint32_t g_aVmcsFields[] =
333	{
334	/* 16-bit control fields. */
335	VMX_VMCS16_VPID,
336	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
337	VMX_VMCS16_EPTP_INDEX,
338	VMX_VMCS16_HLAT_PREFIX_SIZE,
339	VMX_VMCS16_LAST_PID_PTR_INDEX,
340
341	/* 16-bit guest-state fields. */
342	VMX_VMCS16_GUEST_ES_SEL,
343	VMX_VMCS16_GUEST_CS_SEL,
344	VMX_VMCS16_GUEST_SS_SEL,
345	VMX_VMCS16_GUEST_DS_SEL,
346	VMX_VMCS16_GUEST_FS_SEL,
347	VMX_VMCS16_GUEST_GS_SEL,
348	VMX_VMCS16_GUEST_LDTR_SEL,
349	VMX_VMCS16_GUEST_TR_SEL,
350	VMX_VMCS16_GUEST_INTR_STATUS,
351	VMX_VMCS16_GUEST_PML_INDEX,
352	VMX_VMCS16_GUEST_UINV,
353
354	/* 16-bits host-state fields. */
355	VMX_VMCS16_HOST_ES_SEL,
356	VMX_VMCS16_HOST_CS_SEL,
357	VMX_VMCS16_HOST_SS_SEL,
358	VMX_VMCS16_HOST_DS_SEL,
359	VMX_VMCS16_HOST_FS_SEL,
360	VMX_VMCS16_HOST_GS_SEL,
361	VMX_VMCS16_HOST_TR_SEL,
362
363	/* 64-bit control fields. */
364	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
365	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
366	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
367	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
368	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
369	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
370	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
371	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
372	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
373	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
374	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
375	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
376	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
377	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
378	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
379	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
380	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
381	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
382	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
383	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
384	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
385	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
386	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
387	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
388	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
389	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
390	VMX_VMCS64_CTRL_EPTP_FULL,
391	VMX_VMCS64_CTRL_EPTP_HIGH,
392	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
393	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
394	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
395	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
396	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
397	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
398	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
399	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
400	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
401	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
402	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
403	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
404	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
405	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
406	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL,
407	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH,
408	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
409	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
410	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
411	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
412	VMX_VMCS64_CTRL_SPPTP_FULL,
413	VMX_VMCS64_CTRL_SPPTP_HIGH,
414	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
415	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
416	VMX_VMCS64_CTRL_PROC_EXEC3_FULL,
417	VMX_VMCS64_CTRL_PROC_EXEC3_HIGH,
418	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL,
419	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH,
420	VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_FULL,
421	VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_HIGH,
422	VMX_VMCS64_CTRL_HLAT_PTR_FULL,
423	VMX_VMCS64_CTRL_HLAT_PTR_HIGH,
424	VMX_VMCS64_CTRL_EXIT2_FULL,
425	VMX_VMCS64_CTRL_EXIT2_HIGH,
426	VMX_VMCS64_CTRL_SPEC_CTRL_MASK_FULL,
427	VMX_VMCS64_CTRL_SPEC_CTRL_MASK_HIGH,
428	VMX_VMCS64_CTRL_SPEC_CTRL_SHADOW_FULL,
429	VMX_VMCS64_CTRL_SPEC_CTRL_SHADOW_HIGH,
430
431	/* 64-bit read-only data fields. */
432	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
433	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
434
435	/* 64-bit guest-state fields. */
436	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
437	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
438	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
439	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
440	VMX_VMCS64_GUEST_PAT_FULL,
441	VMX_VMCS64_GUEST_PAT_HIGH,
442	VMX_VMCS64_GUEST_EFER_FULL,
443	VMX_VMCS64_GUEST_EFER_HIGH,
444	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
445	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
446	VMX_VMCS64_GUEST_PDPTE0_FULL,
447	VMX_VMCS64_GUEST_PDPTE0_HIGH,
448	VMX_VMCS64_GUEST_PDPTE1_FULL,
449	VMX_VMCS64_GUEST_PDPTE1_HIGH,
450	VMX_VMCS64_GUEST_PDPTE2_FULL,
451	VMX_VMCS64_GUEST_PDPTE2_HIGH,
452	VMX_VMCS64_GUEST_PDPTE3_FULL,
453	VMX_VMCS64_GUEST_PDPTE3_HIGH,
454	VMX_VMCS64_GUEST_BNDCFGS_FULL,
455	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
456	VMX_VMCS64_GUEST_RTIT_CTL_FULL,
457	VMX_VMCS64_GUEST_RTIT_CTL_HIGH,
458	VMX_VMCS64_GUEST_PKRS_FULL,
459	VMX_VMCS64_GUEST_PKRS_HIGH,
460
461	/* 64-bit host-state fields. */
462	VMX_VMCS64_HOST_PAT_FULL,
463	VMX_VMCS64_HOST_PAT_HIGH,
464	VMX_VMCS64_HOST_EFER_FULL,
465	VMX_VMCS64_HOST_EFER_HIGH,
466	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
467	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
468	VMX_VMCS64_HOST_PKRS_FULL,
469	VMX_VMCS64_HOST_PKRS_HIGH,
470
471	/* 32-bit control fields. */
472	VMX_VMCS32_CTRL_PIN_EXEC,
473	VMX_VMCS32_CTRL_PROC_EXEC,
474	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
475	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
476	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
477	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
478	VMX_VMCS32_CTRL_EXIT,
479	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
480	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
481	VMX_VMCS32_CTRL_ENTRY,
482	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
483	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
484	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
485	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
486	VMX_VMCS32_CTRL_TPR_THRESHOLD,
487	VMX_VMCS32_CTRL_PROC_EXEC2,
488	VMX_VMCS32_CTRL_PLE_GAP,
489	VMX_VMCS32_CTRL_PLE_WINDOW,
490	VMX_VMCS32_CTRL_INSTR_TIMEOUT,
491
492	/* 32-bits read-only fields. */
493	VMX_VMCS32_RO_VM_INSTR_ERROR,
494	VMX_VMCS32_RO_EXIT_REASON,
495	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
496	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
497	VMX_VMCS32_RO_IDT_VECTORING_INFO,
498	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
499	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
500	VMX_VMCS32_RO_EXIT_INSTR_INFO,
501
502	/* 32-bit guest-state fields. */
503	VMX_VMCS32_GUEST_ES_LIMIT,
504	VMX_VMCS32_GUEST_CS_LIMIT,
505	VMX_VMCS32_GUEST_SS_LIMIT,
506	VMX_VMCS32_GUEST_DS_LIMIT,
507	VMX_VMCS32_GUEST_FS_LIMIT,
508	VMX_VMCS32_GUEST_GS_LIMIT,
509	VMX_VMCS32_GUEST_LDTR_LIMIT,
510	VMX_VMCS32_GUEST_TR_LIMIT,
511	VMX_VMCS32_GUEST_GDTR_LIMIT,
512	VMX_VMCS32_GUEST_IDTR_LIMIT,
513	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
514	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
515	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
516	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
517	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
518	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
519	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
520	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
521	VMX_VMCS32_GUEST_INT_STATE,
522	VMX_VMCS32_GUEST_ACTIVITY_STATE,
523	VMX_VMCS32_GUEST_SMBASE,
524	VMX_VMCS32_GUEST_SYSENTER_CS,
525	VMX_VMCS32_PREEMPT_TIMER_VALUE,
526
527	/* 32-bit host-state fields. */
528	VMX_VMCS32_HOST_SYSENTER_CS,
529
530	/* Natural-width control fields. */
531	VMX_VMCS_CTRL_CR0_MASK,
532	VMX_VMCS_CTRL_CR4_MASK,
533	VMX_VMCS_CTRL_CR0_READ_SHADOW,
534	VMX_VMCS_CTRL_CR4_READ_SHADOW,
535	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
536	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
537	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
538	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
539
540	/* Natural-width read-only data fields. */
541	VMX_VMCS_RO_EXIT_QUALIFICATION,
542	VMX_VMCS_RO_IO_RCX,
543	VMX_VMCS_RO_IO_RSI,
544	VMX_VMCS_RO_IO_RDI,
545	VMX_VMCS_RO_IO_RIP,
546	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
547
548	/* Natural-width guest-state field */
549	VMX_VMCS_GUEST_CR0,
550	VMX_VMCS_GUEST_CR3,
551	VMX_VMCS_GUEST_CR4,
552	VMX_VMCS_GUEST_ES_BASE,
553	VMX_VMCS_GUEST_CS_BASE,
554	VMX_VMCS_GUEST_SS_BASE,
555	VMX_VMCS_GUEST_DS_BASE,
556	VMX_VMCS_GUEST_FS_BASE,
557	VMX_VMCS_GUEST_GS_BASE,
558	VMX_VMCS_GUEST_LDTR_BASE,
559	VMX_VMCS_GUEST_TR_BASE,
560	VMX_VMCS_GUEST_GDTR_BASE,
561	VMX_VMCS_GUEST_IDTR_BASE,
562	VMX_VMCS_GUEST_DR7,
563	VMX_VMCS_GUEST_RSP,
564	VMX_VMCS_GUEST_RIP,
565	VMX_VMCS_GUEST_RFLAGS,
566	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
567	VMX_VMCS_GUEST_SYSENTER_ESP,
568	VMX_VMCS_GUEST_SYSENTER_EIP,
569	VMX_VMCS_GUEST_S_CET,
570	VMX_VMCS_GUEST_SSP,
571	VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR,
572
573	/* Natural-width host-state fields */
574	VMX_VMCS_HOST_CR0,
575	VMX_VMCS_HOST_CR3,
576	VMX_VMCS_HOST_CR4,
577	VMX_VMCS_HOST_FS_BASE,
578	VMX_VMCS_HOST_GS_BASE,
579	VMX_VMCS_HOST_TR_BASE,
580	VMX_VMCS_HOST_GDTR_BASE,
581	VMX_VMCS_HOST_IDTR_BASE,
582	VMX_VMCS_HOST_SYSENTER_ESP,
583	VMX_VMCS_HOST_SYSENTER_EIP,
584	VMX_VMCS_HOST_RSP,
585	VMX_VMCS_HOST_RIP,
586	VMX_VMCS_HOST_S_CET,
587	VMX_VMCS_HOST_SSP,
588	VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR
589	};
590	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
591
592	#ifdef HMVMX_USE_FUNCTION_TABLE
593	/**
594	* VMX_EXIT dispatch table.
595	*/
596	static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
597	{
598	/* 0 VMX_EXIT_XCPT_OR_NMI */ { vmxHCExitXcptOrNmi },
599	/* 1 VMX_EXIT_EXT_INT */ { vmxHCExitExtInt },
600	/* 2 VMX_EXIT_TRIPLE_FAULT */ { vmxHCExitTripleFault },
601	/* 3 VMX_EXIT_INIT_SIGNAL */ { vmxHCExitErrUnexpected },
602	/* 4 VMX_EXIT_SIPI */ { vmxHCExitErrUnexpected },
603	/* 5 VMX_EXIT_IO_SMI */ { vmxHCExitErrUnexpected },
604	/* 6 VMX_EXIT_SMI */ { vmxHCExitErrUnexpected },
605	/* 7 VMX_EXIT_INT_WINDOW */ { vmxHCExitIntWindow },
606	/* 8 VMX_EXIT_NMI_WINDOW */ { vmxHCExitNmiWindow },
607	/* 9 VMX_EXIT_TASK_SWITCH */ { vmxHCExitTaskSwitch },
608	/* 10 VMX_EXIT_CPUID */ { vmxHCExitCpuid },
609	/* 11 VMX_EXIT_GETSEC */ { vmxHCExitGetsec },
610	/* 12 VMX_EXIT_HLT */ { vmxHCExitHlt },
611	/* 13 VMX_EXIT_INVD */ { vmxHCExitInvd },
612	/* 14 VMX_EXIT_INVLPG */ { vmxHCExitInvlpg },
613	/* 15 VMX_EXIT_RDPMC */ { vmxHCExitRdpmc },
614	/* 16 VMX_EXIT_RDTSC */ { vmxHCExitRdtsc },
615	/* 17 VMX_EXIT_RSM */ { vmxHCExitErrUnexpected },
616	/* 18 VMX_EXIT_VMCALL */ { vmxHCExitVmcall },
617	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
618	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitVmclear },
619	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitVmlaunch },
620	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitVmptrld },
621	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitVmptrst },
622	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitVmread },
623	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitVmresume },
624	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitVmwrite },
625	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitVmxoff },
626	/* 27 VMX_EXIT_VMXON */ { vmxHCExitVmxon },
627	#else
628	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitSetPendingXcptUD },
629	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitSetPendingXcptUD },
630	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitSetPendingXcptUD },
631	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitSetPendingXcptUD },
632	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitSetPendingXcptUD },
633	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitSetPendingXcptUD },
634	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitSetPendingXcptUD },
635	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitSetPendingXcptUD },
636	/* 27 VMX_EXIT_VMXON */ { vmxHCExitSetPendingXcptUD },
637	#endif
638	/* 28 VMX_EXIT_MOV_CRX */ { vmxHCExitMovCRx },
639	/* 29 VMX_EXIT_MOV_DRX */ { vmxHCExitMovDRx },
640	/* 30 VMX_EXIT_IO_INSTR */ { vmxHCExitIoInstr },
641	/* 31 VMX_EXIT_RDMSR */ { vmxHCExitRdmsr },
642	/* 32 VMX_EXIT_WRMSR */ { vmxHCExitWrmsr },
643	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ { vmxHCExitErrInvalidGuestState },
644	/* 34 VMX_EXIT_ERR_MSR_LOAD */ { vmxHCExitErrUnexpected },
645	/* 35 UNDEFINED */ { vmxHCExitErrUnexpected },
646	/* 36 VMX_EXIT_MWAIT */ { vmxHCExitMwait },
647	/* 37 VMX_EXIT_MTF */ { vmxHCExitMtf },
648	/* 38 UNDEFINED */ { vmxHCExitErrUnexpected },
649	/* 39 VMX_EXIT_MONITOR */ { vmxHCExitMonitor },
650	/* 40 VMX_EXIT_PAUSE */ { vmxHCExitPause },
651	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ { vmxHCExitErrUnexpected },
652	/* 42 UNDEFINED */ { vmxHCExitErrUnexpected },
653	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ { vmxHCExitTprBelowThreshold },
654	/* 44 VMX_EXIT_APIC_ACCESS */ { vmxHCExitApicAccess },
655	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ { vmxHCExitErrUnexpected },
656	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ { vmxHCExitErrUnexpected },
657	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ { vmxHCExitErrUnexpected },
658	/* 48 VMX_EXIT_EPT_VIOLATION */ { vmxHCExitEptViolation },
659	/* 49 VMX_EXIT_EPT_MISCONFIG */ { vmxHCExitEptMisconfig },
660	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
661	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitInvept },
662	#else
663	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitSetPendingXcptUD },
664	#endif
665	/* 51 VMX_EXIT_RDTSCP */ { vmxHCExitRdtscp },
666	/* 52 VMX_EXIT_PREEMPT_TIMER */ { vmxHCExitPreemptTimer },
667	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
668	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitInvvpid },
669	#else
670	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitSetPendingXcptUD },
671	#endif
672	/* 54 VMX_EXIT_WBINVD */ { vmxHCExitWbinvd },
673	/* 55 VMX_EXIT_XSETBV */ { vmxHCExitXsetbv },
674	/* 56 VMX_EXIT_APIC_WRITE */ { vmxHCExitErrUnexpected },
675	/* 57 VMX_EXIT_RDRAND */ { vmxHCExitErrUnexpected },
676	/* 58 VMX_EXIT_INVPCID */ { vmxHCExitInvpcid },
677	/* 59 VMX_EXIT_VMFUNC */ { vmxHCExitErrUnexpected },
678	/* 60 VMX_EXIT_ENCLS */ { vmxHCExitErrUnexpected },
679	/* 61 VMX_EXIT_RDSEED */ { vmxHCExitErrUnexpected },
680	/* 62 VMX_EXIT_PML_FULL */ { vmxHCExitErrUnexpected },
681	/* 63 VMX_EXIT_XSAVES */ { vmxHCExitErrUnexpected },
682	/* 64 VMX_EXIT_XRSTORS */ { vmxHCExitErrUnexpected },
683	/* 65 UNDEFINED */ { vmxHCExitErrUnexpected },
684	/* 66 VMX_EXIT_SPP_EVENT */ { vmxHCExitErrUnexpected },
685	/* 67 VMX_EXIT_UMWAIT */ { vmxHCExitErrUnexpected },
686	/* 68 VMX_EXIT_TPAUSE */ { vmxHCExitErrUnexpected },
687	/* 69 VMX_EXIT_LOADIWKEY */ { vmxHCExitErrUnexpected },
688	};
689	#endif /* HMVMX_USE_FUNCTION_TABLE */
690
691	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
692	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
693	{
694	/* 0 */ "(Not Used)",
695	/* 1 */ "VMCALL executed in VMX root operation.",
696	/* 2 */ "VMCLEAR with invalid physical address.",
697	/* 3 */ "VMCLEAR with VMXON pointer.",
698	/* 4 */ "VMLAUNCH with non-clear VMCS.",
699	/* 5 */ "VMRESUME with non-launched VMCS.",
700	/* 6 */ "VMRESUME after VMXOFF",
701	/* 7 */ "VM-entry with invalid control fields.",
702	/* 8 */ "VM-entry with invalid host state fields.",
703	/* 9 */ "VMPTRLD with invalid physical address.",
704	/* 10 */ "VMPTRLD with VMXON pointer.",
705	/* 11 */ "VMPTRLD with incorrect revision identifier.",
706	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
707	/* 13 */ "VMWRITE to read-only VMCS component.",
708	/* 14 */ "(Not Used)",
709	/* 15 */ "VMXON executed in VMX root operation.",
710	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
711	/* 17 */ "VM-entry with non-launched executing VMCS.",
712	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
713	/* 19 */ "VMCALL with non-clear VMCS.",
714	/* 20 */ "VMCALL with invalid VM-exit control fields.",
715	/* 21 */ "(Not Used)",
716	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
717	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
718	/* 24 */ "VMCALL with invalid SMM-monitor features.",
719	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
720	/* 26 */ "VM-entry with events blocked by MOV SS.",
721	/* 27 */ "(Not Used)",
722	/* 28 */ "Invalid operand to INVEPT/INVVPID."
723	};
724	#endif /* VBOX_STRICT && LOG_ENABLED */
725
726
727	/**
728	* Gets the CR0 guest/host mask.
729	*
730	* These bits typically does not change through the lifetime of a VM. Any bit set in
731	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
732	* by the guest.
733	*
734	* @returns The CR0 guest/host mask.
735	* @param pVCpu The cross context virtual CPU structure.
736	*/
737	static uint64_t vmxHCGetFixedCr0Mask(PCVMCPUCC pVCpu)
738	{
739	/*
740	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
741	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
742	*
743	* Furthermore, modifications to any bits that are reserved/unspecified currently
744	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
745	* when future CPUs specify and use currently reserved/unspecified bits.
746	*/
747	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
748	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
749	* and @bugref{6944}. */
750	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
751	AssertCompile(RT_HI_U32(VMX_EXIT_HOST_CR0_IGNORE_MASK) == UINT32_C(0xffffffff)); /* Paranoia. */
752	return ( X86_CR0_PE
753	\| X86_CR0_NE
754	\| (VM_IS_VMX_NESTED_PAGING(pVM) ? 0 : X86_CR0_WP)
755	\| X86_CR0_PG
756	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
757	}
758
759
760	/**
761	* Gets the CR4 guest/host mask.
762	*
763	* These bits typically does not change through the lifetime of a VM. Any bit set in
764	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
765	* by the guest.
766	*
767	* @returns The CR4 guest/host mask.
768	* @param pVCpu The cross context virtual CPU structure.
769	*/
770	static uint64_t vmxHCGetFixedCr4Mask(PCVMCPUCC pVCpu)
771	{
772	/*
773	* We construct a mask of all CR4 bits that the guest can modify without causing
774	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
775	* a VM-exit when the guest attempts to modify them when executing using
776	* hardware-assisted VMX.
777	*
778	* When a feature is not exposed to the guest (and may be present on the host),
779	* we want to intercept guest modifications to the bit so we can emulate proper
780	* behavior (e.g., #GP).
781	*
782	* Furthermore, only modifications to those bits that don't require immediate
783	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
784	* depends on CR3 which might not always be the guest value while executing
785	* using hardware-assisted VMX.
786	*/
787	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
788	bool fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
789	#ifdef IN_NEM_DARWIN
790	bool fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
791	#endif
792	bool fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
793
794	/*
795	* Paranoia.
796	* Ensure features exposed to the guest are present on the host.
797	*/
798	AssertStmt(!fFsGsBase \|\| g_CpumHostFeatures.s.fFsGsBase, fFsGsBase = 0);
799	#ifdef IN_NEM_DARWIN
800	AssertStmt(!fXSaveRstor \|\| g_CpumHostFeatures.s.fXSaveRstor, fXSaveRstor = 0);
801	#endif
802	AssertStmt(!fFxSaveRstor \|\| g_CpumHostFeatures.s.fFxSaveRstor, fFxSaveRstor = 0);
803
804	uint64_t const fGstMask = X86_CR4_PVI
805	\| X86_CR4_TSD
806	\| X86_CR4_DE
807	\| X86_CR4_MCE
808	\| X86_CR4_PCE
809	\| X86_CR4_OSXMMEEXCPT
810	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
811	#ifdef IN_NEM_DARWIN /* On native VT-x setting OSXSAVE must exit as we need to load guest XCR0 (see
812	fLoadSaveGuestXcr0). These exits are not needed on Darwin as that's not our problem. */
813	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
814	#endif
815	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0);
816	return ~fGstMask;
817	}
818
819
820	/**
821	* Checks whether an \#AC exception generated while executing a guest (or
822	* nested-guest) was due to a split-lock memory access.
823	*
824	* @returns @c true if split-lock triggered the \#AC, @c false otherwise.
825	* @param pVCpu The cross context virtual CPU structure.
826	*/
827	DECL_FORCE_INLINE(bool) vmxHCIsSplitLockAcXcpt(PVMCPU pVCpu)
828	{
829	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
830	if ( !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM) /* 1. If 486-style alignment checks aren't enabled, this must be a split-lock #AC. */
831	\|\| !(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_AC) /* 2. When the EFLAGS.AC != 0 this can only be a split-lock case. */
832	\|\| CPUMGetGuestCPL(pVCpu) != 3) /* 3. #AC cannot happen in rings 0-2 except for split-lock detection. */
833	return true;
834	return false;
835	}
836
837
838	/**
839	* Adds one or more exceptions to the exception bitmap and commits it to the current
840	* VMCS.
841	*
842	* @param pVCpu The cross context virtual CPU structure.
843	* @param pVmxTransient The VMX-transient structure.
844	* @param uXcptMask The exception(s) to add.
845	*/
846	static void vmxHCAddXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
847	{
848	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
849	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
850	if ((uXcptBitmap & uXcptMask) != uXcptMask)
851	{
852	uXcptBitmap \|= uXcptMask;
853	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
854	AssertRC(rc);
855	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
856	}
857	}
858
859
860	/**
861	* Adds an exception to the exception bitmap and commits it to the current VMCS.
862	*
863	* @param pVCpu The cross context virtual CPU structure.
864	* @param pVmxTransient The VMX-transient structure.
865	* @param uXcpt The exception to add.
866	*/
867	static void vmxHCAddXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
868	{
869	Assert(uXcpt <= X86_XCPT_LAST);
870	vmxHCAddXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT_32(uXcpt));
871	}
872
873
874	/**
875	* Remove one or more exceptions from the exception bitmap and commits it to the
876	* current VMCS.
877	*
878	* This takes care of not removing the exception intercept if a nested-guest
879	* requires the exception to be intercepted.
880	*
881	* @returns VBox status code.
882	* @param pVCpu The cross context virtual CPU structure.
883	* @param pVmxTransient The VMX-transient structure.
884	* @param uXcptMask The exception(s) to remove.
885	*/
886	static int vmxHCRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
887	{
888	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
889	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
890	if (uXcptBitmap & uXcptMask)
891	{
892	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
893	if (!pVmxTransient->fIsNestedGuest)
894	{ /* likely */ }
895	else
896	uXcptMask &= ~pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap;
897	#endif
898	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
899	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
900	\| RT_BIT(X86_XCPT_DE)
901	\| RT_BIT(X86_XCPT_NM)
902	\| RT_BIT(X86_XCPT_TS)
903	\| RT_BIT(X86_XCPT_UD)
904	\| RT_BIT(X86_XCPT_NP)
905	\| RT_BIT(X86_XCPT_SS)
906	\| RT_BIT(X86_XCPT_GP)
907	\| RT_BIT(X86_XCPT_PF)
908	\| RT_BIT(X86_XCPT_MF));
909	#elif defined(HMVMX_ALWAYS_TRAP_PF)
910	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
911	#endif
912	if (uXcptMask)
913	{
914	/* Validate we are not removing any essential exception intercepts. */
915	#ifndef IN_NEM_DARWIN
916	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
917	#else
918	Assert(!(uXcptMask & RT_BIT(X86_XCPT_PF)));
919	#endif
920	NOREF(pVCpu);
921	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
922	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
923
924	/* Remove it from the exception bitmap. */
925	uXcptBitmap &= ~uXcptMask;
926
927	/* Commit and update the cache if necessary. */
928	if (pVmcsInfo->u32XcptBitmap != uXcptBitmap)
929	{
930	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
931	AssertRC(rc);
932	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
933	}
934	}
935	}
936	return VINF_SUCCESS;
937	}
938
939
940	/**
941	* Remove an exceptions from the exception bitmap and commits it to the current
942	* VMCS.
943	*
944	* @returns VBox status code.
945	* @param pVCpu The cross context virtual CPU structure.
946	* @param pVmxTransient The VMX-transient structure.
947	* @param uXcpt The exception to remove.
948	*/
949	static int vmxHCRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
950	{
951	return vmxHCRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
952	}
953
954	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
955
956	/**
957	* Loads the shadow VMCS specified by the VMCS info. object.
958	*
959	* @returns VBox status code.
960	* @param pVmcsInfo The VMCS info. object.
961	*
962	* @remarks Can be called with interrupts disabled.
963	*/
964	static int vmxHCLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
965	{
966	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
967	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
968
969	return VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
970	}
971
972
973	/**
974	* Clears the shadow VMCS specified by the VMCS info. object.
975	*
976	* @returns VBox status code.
977	* @param pVmcsInfo The VMCS info. object.
978	*
979	* @remarks Can be called with interrupts disabled.
980	*/
981	static int vmxHCClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
982	{
983	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
984	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
985
986	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
987	if (RT_SUCCESS(rc))
988	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
989	return rc;
990	}
991
992
993	/**
994	* Switches from and to the specified VMCSes.
995	*
996	* @returns VBox status code.
997	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
998	* @param pVmcsInfoTo The VMCS info. object we are switching to.
999	*
1000	* @remarks Called with interrupts disabled.
1001	*/
1002	static int vmxHCSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1003	{
1004	/*
1005	* Clear the VMCS we are switching out if it has not already been cleared.
1006	* This will sync any CPU internal data back to the VMCS.
1007	*/
1008	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1009	{
1010	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1011	if (RT_SUCCESS(rc))
1012	{
1013	/*
1014	* The shadow VMCS, if any, would not be active at this point since we
1015	* would have cleared it while importing the virtual hardware-virtualization
1016	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1017	* clear the shadow VMCS here, just assert for safety.
1018	*/
1019	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1020	}
1021	else
1022	return rc;
1023	}
1024
1025	/*
1026	* Clear the VMCS we are switching to if it has not already been cleared.
1027	* This will initialize the VMCS launch state to "clear" required for loading it.
1028	*
1029	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1030	*/
1031	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1032	{
1033	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1034	if (RT_SUCCESS(rc))
1035	{ /* likely */ }
1036	else
1037	return rc;
1038	}
1039
1040	/*
1041	* Finally, load the VMCS we are switching to.
1042	*/
1043	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1044	}
1045
1046
1047	/**
1048	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1049	* caller.
1050	*
1051	* @returns VBox status code.
1052	* @param pVCpu The cross context virtual CPU structure.
1053	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1054	* true) or guest VMCS (pass false).
1055	*/
1056	static int vmxHCSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1057	{
1058	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1059	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1060
1061	PVMXVMCSINFO pVmcsInfoFrom;
1062	PVMXVMCSINFO pVmcsInfoTo;
1063	if (fSwitchToNstGstVmcs)
1064	{
1065	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfo;
1066	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1067	Assert(!pVCpu->hm.s.vmx.fMergedNstGstCtls);
1068	}
1069	else
1070	{
1071	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1072	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfo;
1073	}
1074
1075	/*
1076	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1077	* preemption hook code path acquires the current VMCS.
1078	*/
1079	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1080
1081	int rc = vmxHCSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1082	if (RT_SUCCESS(rc))
1083	{
1084	pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1085	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3 = fSwitchToNstGstVmcs;
1086
1087	/*
1088	* If we are switching to a VMCS that was executed on a different host CPU or was
1089	* never executed before, flag that we need to export the host state before executing
1090	* guest/nested-guest code using hardware-assisted VMX.
1091	*
1092	* This could probably be done in a preemptible context since the preemption hook
1093	* will flag the necessary change in host context. However, since preemption is
1094	* already disabled and to avoid making assumptions about host specific code in
1095	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1096	* disabled.
1097	*/
1098	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1099	{ /* likely */ }
1100	else
1101	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1102
1103	ASMSetFlags(fEFlags);
1104
1105	/*
1106	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1107	* flag that we need to update the host MSR values there. Even if we decide in the
1108	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1109	* if its content differs, we would have to update the host MSRs anyway.
1110	*/
1111	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
1112	}
1113	else
1114	ASMSetFlags(fEFlags);
1115	return rc;
1116	}
1117
1118	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1119	#ifdef VBOX_STRICT
1120
1121	/**
1122	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1123	* transient structure.
1124	*
1125	* @param pVCpu The cross context virtual CPU structure.
1126	* @param pVmxTransient The VMX-transient structure.
1127	*/
1128	DECLINLINE(void) vmxHCReadEntryIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1129	{
1130	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1131	AssertRC(rc);
1132	}
1133
1134
1135	/**
1136	* Reads the VM-entry exception error code field from the VMCS into
1137	* the VMX transient structure.
1138	*
1139	* @param pVCpu The cross context virtual CPU structure.
1140	* @param pVmxTransient The VMX-transient structure.
1141	*/
1142	DECLINLINE(void) vmxHCReadEntryXcptErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1143	{
1144	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1145	AssertRC(rc);
1146	}
1147
1148
1149	/**
1150	* Reads the VM-entry exception error code field from the VMCS into
1151	* the VMX transient structure.
1152	*
1153	* @param pVCpu The cross context virtual CPU structure.
1154	* @param pVmxTransient The VMX-transient structure.
1155	*/
1156	DECLINLINE(void) vmxHCReadEntryInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1157	{
1158	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1159	AssertRC(rc);
1160	}
1161
1162	#endif /* VBOX_STRICT */
1163
1164
1165	/**
1166	* Reads VMCS fields into the VMXTRANSIENT structure, slow path version.
1167	*
1168	* Don't call directly unless the it's likely that some or all of the fields
1169	* given in @a a_fReadMask have already been read.
1170	*
1171	* @tparam a_fReadMask The fields to read.
1172	* @param pVCpu The cross context virtual CPU structure.
1173	* @param pVmxTransient The VMX-transient structure.
1174	*/
1175	template<uint32_t const a_fReadMask>
1176	static void vmxHCReadToTransientSlow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1177	{
1178	AssertCompile((a_fReadMask & ~( HMVMX_READ_EXIT_QUALIFICATION
1179	\| HMVMX_READ_EXIT_INSTR_LEN
1180	\| HMVMX_READ_EXIT_INSTR_INFO
1181	\| HMVMX_READ_IDT_VECTORING_INFO
1182	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1183	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1184	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1185	\| HMVMX_READ_GUEST_LINEAR_ADDR
1186	\| HMVMX_READ_GUEST_PHYSICAL_ADDR
1187	\| HMVMX_READ_GUEST_PENDING_DBG_XCPTS
1188	)) == 0);
1189
1190	if ((pVmxTransient->fVmcsFieldsRead & a_fReadMask) != a_fReadMask)
1191	{
1192	uint32_t const fVmcsFieldsRead = pVmxTransient->fVmcsFieldsRead;
1193
1194	if ( (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION)
1195	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1196	{
1197	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1198	AssertRC(rc);
1199	}
1200	if ( (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN)
1201	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1202	{
1203	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1204	AssertRC(rc);
1205	}
1206	if ( (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO)
1207	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1208	{
1209	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1210	AssertRC(rc);
1211	}
1212	if ( (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO)
1213	&& !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1214	{
1215	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1216	AssertRC(rc);
1217	}
1218	if ( (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
1219	&& !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1220	{
1221	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1222	AssertRC(rc);
1223	}
1224	if ( (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO)
1225	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1226	{
1227	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1228	AssertRC(rc);
1229	}
1230	if ( (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
1231	&& !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1232	{
1233	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1234	AssertRC(rc);
1235	}
1236	if ( (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR)
1237	&& !(fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1238	{
1239	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1240	AssertRC(rc);
1241	}
1242	if ( (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR)
1243	&& !(fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1244	{
1245	int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1246	AssertRC(rc);
1247	}
1248	if ( (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
1249	&& !(fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1250	{
1251	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1252	AssertRC(rc);
1253	}
1254
1255	pVmxTransient->fVmcsFieldsRead \|= a_fReadMask;
1256	}
1257	}
1258
1259
1260	/**
1261	* Reads VMCS fields into the VMXTRANSIENT structure.
1262	*
1263	* This optimizes for the case where none of @a a_fReadMask has been read yet,
1264	* generating an optimized read sequences w/o any conditionals between in
1265	* non-strict builds.
1266	*
1267	* @tparam a_fReadMask The fields to read. One or more of the
1268	* HMVMX_READ_XXX fields ORed together.
1269	* @param pVCpu The cross context virtual CPU structure.
1270	* @param pVmxTransient The VMX-transient structure.
1271	*/
1272	template<uint32_t const a_fReadMask>
1273	DECLINLINE(void) vmxHCReadToTransient(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1274	{
1275	AssertCompile((a_fReadMask & ~( HMVMX_READ_EXIT_QUALIFICATION
1276	\| HMVMX_READ_EXIT_INSTR_LEN
1277	\| HMVMX_READ_EXIT_INSTR_INFO
1278	\| HMVMX_READ_IDT_VECTORING_INFO
1279	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1280	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1281	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1282	\| HMVMX_READ_GUEST_LINEAR_ADDR
1283	\| HMVMX_READ_GUEST_PHYSICAL_ADDR
1284	\| HMVMX_READ_GUEST_PENDING_DBG_XCPTS
1285	)) == 0);
1286
1287	if (RT_LIKELY(!(pVmxTransient->fVmcsFieldsRead & a_fReadMask)))
1288	{
1289	if (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION)
1290	{
1291	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1292	AssertRC(rc);
1293	}
1294	if (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN)
1295	{
1296	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1297	AssertRC(rc);
1298	}
1299	if (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO)
1300	{
1301	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1302	AssertRC(rc);
1303	}
1304	if (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO)
1305	{
1306	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1307	AssertRC(rc);
1308	}
1309	if (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
1310	{
1311	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1312	AssertRC(rc);
1313	}
1314	if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO)
1315	{
1316	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1317	AssertRC(rc);
1318	}
1319	if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
1320	{
1321	int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1322	AssertRC(rc);
1323	}
1324	if (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR)
1325	{
1326	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1327	AssertRC(rc);
1328	}
1329	if (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR)
1330	{
1331	int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1332	AssertRC(rc);
1333	}
1334	if (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
1335	{
1336	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1337	AssertRC(rc);
1338	}
1339
1340	pVmxTransient->fVmcsFieldsRead \|= a_fReadMask;
1341	}
1342	else
1343	{
1344	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatReadToTransientFallback);
1345	Log11Func(("a_fReadMask=%#x fVmcsFieldsRead=%#x => %#x - Taking inefficient code path!\n",
1346	a_fReadMask, pVmxTransient->fVmcsFieldsRead, a_fReadMask & pVmxTransient->fVmcsFieldsRead));
1347	vmxHCReadToTransientSlow<a_fReadMask>(pVCpu, pVmxTransient);
1348	}
1349	}
1350
1351
1352	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1353	/**
1354	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1355	*
1356	* @param pVCpu The cross context virtual CPU structure.
1357	* @param pVmxTransient The VMX-transient structure.
1358	*/
1359	static void vmxHCReadAllRoFieldsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1360	{
1361	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1362	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1363	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1364	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1365	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1366	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1367	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1368	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1369	rc \|= VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1370	AssertRC(rc);
1371	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1372	\| HMVMX_READ_EXIT_INSTR_LEN
1373	\| HMVMX_READ_EXIT_INSTR_INFO
1374	\| HMVMX_READ_IDT_VECTORING_INFO
1375	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1376	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1377	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1378	\| HMVMX_READ_GUEST_LINEAR_ADDR
1379	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1380	}
1381	#endif
1382
1383	/**
1384	* Verifies that our cached values of the VMCS fields are all consistent with
1385	* what's actually present in the VMCS.
1386	*
1387	* @returns VBox status code.
1388	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1389	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1390	* VMCS content. HMCPU error-field is
1391	* updated, see VMX_VCI_XXX.
1392	* @param pVCpu The cross context virtual CPU structure.
1393	* @param pVmcsInfo The VMCS info. object.
1394	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1395	*/
1396	static int vmxHCCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1397	{
1398	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
1399
1400	uint32_t u32Val;
1401	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
1402	AssertRC(rc);
1403	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
1404	("%s entry controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
1405	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_ENTRY,
1406	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1407
1408	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXIT, &u32Val);
1409	AssertRC(rc);
1410	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
1411	("%s exit controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
1412	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_EXIT,
1413	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1414
1415	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1416	AssertRC(rc);
1417	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
1418	("%s pin controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
1419	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1420	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1421
1422	/** @todo Currently disabled for nested-guests because we run into bit differences
1423	* with for INT_WINDOW, RDTSC/P, see @bugref{10318}. Later try figure out
1424	* why and re-enable. */
1425	if (!fIsNstGstVmcs)
1426	{
1427	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1428	AssertRC(rc);
1429	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
1430	("%s proc controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
1431	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1432	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1433	}
1434
1435	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1436	{
1437	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1438	AssertRC(rc);
1439	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
1440	("%s proc2 controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
1441	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1442	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1443	}
1444
1445	uint64_t u64Val;
1446	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS)
1447	{
1448	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_PROC_EXEC3_FULL, &u64Val);
1449	AssertRC(rc);
1450	AssertMsgReturnStmt(pVmcsInfo->u64ProcCtls3 == u64Val,
1451	("%s proc3 controls mismatch: Cache=%#RX32 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64ProcCtls3, u64Val),
1452	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC3,
1453	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1454	}
1455
1456	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1457	AssertRC(rc);
1458	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
1459	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
1460	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1461	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1462
1463	/*
1464	* The TSC offset will only be used when RDTSC is not intercepted.
1465	* Since we don't actively clear it while switching between intercepting or not,
1466	* the value here could be stale.
1467	*/
1468	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
1469	{
1470	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1471	AssertRC(rc);
1472	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
1473	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
1474	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1475	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1476	}
1477
1478	NOREF(pcszVmcs);
1479	return VINF_SUCCESS;
1480	}
1481
1482
1483	/**
1484	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
1485	* VMCS.
1486	*
1487	* This is typically required when the guest changes paging mode.
1488	*
1489	* @returns VBox status code.
1490	* @param pVCpu The cross context virtual CPU structure.
1491	* @param pVmxTransient The VMX-transient structure.
1492	*
1493	* @remarks Requires EFER.
1494	* @remarks No-long-jump zone!!!
1495	*/
1496	static int vmxHCExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1497	{
1498	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
1499	{
1500	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1501	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1502
1503	/*
1504	* VM-entry controls.
1505	*/
1506	{
1507	uint32_t fVal = g_HmMsrs.u.vmx.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
1508	uint32_t const fZap = g_HmMsrs.u.vmx.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1509
1510	/*
1511	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
1512	* The first VT-x capable CPUs only supported the 1-setting of this bit.
1513	*
1514	* For nested-guests, this is a mandatory VM-entry control. It's also
1515	* required because we do not want to leak host bits to the nested-guest.
1516	*/
1517	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
1518
1519	/*
1520	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
1521	*
1522	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
1523	* required to get the nested-guest working with hardware-assisted VMX execution.
1524	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
1525	* can skip intercepting changes to the EFER MSR. This is why it needs to be done
1526	* here rather than while merging the guest VMCS controls.
1527	*/
1528	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
1529	{
1530	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
1531	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
1532	}
1533	else
1534	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
1535
1536	/*
1537	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
1538	*
1539	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
1540	* regardless of whether the nested-guest VMCS specifies it because we are free to
1541	* load whatever MSRs we require and we do not need to modify the guest visible copy
1542	* of the VM-entry MSR load area.
1543	*/
1544	if ( g_fHmVmxSupportsVmcsEfer
1545	#ifndef IN_NEM_DARWIN
1546	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient)
1547	#endif
1548	)
1549	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
1550	else
1551	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
1552
1553	/*
1554	* The following should -not- be set (since we're not in SMM mode):
1555	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
1556	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
1557	*/
1558
1559	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
1560	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
1561
1562	if ((fVal & fZap) == fVal)
1563	{ /* likely */ }
1564	else
1565	{
1566	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1567	g_HmMsrs.u.vmx.EntryCtls.n.allowed0, fVal, fZap));
1568	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_ENTRY;
1569	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1570	}
1571
1572	/* Commit it to the VMCS. */
1573	if (pVmcsInfo->u32EntryCtls != fVal)
1574	{
1575	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, fVal);
1576	AssertRC(rc);
1577	pVmcsInfo->u32EntryCtls = fVal;
1578	}
1579	}
1580
1581	/*
1582	* VM-exit controls.
1583	*/
1584	{
1585	uint32_t fVal = g_HmMsrs.u.vmx.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
1586	uint32_t const fZap = g_HmMsrs.u.vmx.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1587
1588	/*
1589	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
1590	* supported the 1-setting of this bit.
1591	*
1592	* For nested-guests, we set the "save debug controls" as the converse
1593	* "load debug controls" is mandatory for nested-guests anyway.
1594	*/
1595	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
1596
1597	/*
1598	* Set the host long mode active (EFER.LMA) bit (which Intel calls
1599	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
1600	* host EFER.LMA and EFER.LME bit to this value. See assertion in
1601	* vmxHCExportHostMsrs().
1602	*
1603	* For nested-guests, we always set this bit as we do not support 32-bit
1604	* hosts.
1605	*/
1606	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
1607
1608	#ifndef IN_NEM_DARWIN
1609	/*
1610	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
1611	*
1612	* For nested-guests, we should use the "save IA32_EFER" control if we also
1613	* used the "load IA32_EFER" control while exporting VM-entry controls.
1614	*/
1615	if ( g_fHmVmxSupportsVmcsEfer
1616	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
1617	{
1618	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
1619	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
1620	}
1621	#endif
1622
1623	/*
1624	* Enable saving of the VMX-preemption timer value on VM-exit.
1625	* For nested-guests, currently not exposed/used.
1626	*/
1627	/** @todo r=bird: Measure performance hit because of this vs. always rewriting
1628	* the timer value. */
1629	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
1630	{
1631	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
1632	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
1633	}
1634
1635	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
1636	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
1637
1638	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
1639	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
1640	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
1641
1642	if ((fVal & fZap) == fVal)
1643	{ /* likely */ }
1644	else
1645	{
1646	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1647	g_HmMsrs.u.vmx.ExitCtls.n.allowed0, fVal, fZap));
1648	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_EXIT;
1649	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1650	}
1651
1652	/* Commit it to the VMCS. */
1653	if (pVmcsInfo->u32ExitCtls != fVal)
1654	{
1655	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT, fVal);
1656	AssertRC(rc);
1657	pVmcsInfo->u32ExitCtls = fVal;
1658	}
1659	}
1660
1661	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
1662	}
1663	return VINF_SUCCESS;
1664	}
1665
1666
1667	/**
1668	* Sets the TPR threshold in the VMCS.
1669	*
1670	* @param pVCpu The cross context virtual CPU structure.
1671	* @param pVmcsInfo The VMCS info. object.
1672	* @param u32TprThreshold The TPR threshold (task-priority class only).
1673	*/
1674	DECLINLINE(void) vmxHCApicSetTprThreshold(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
1675	{
1676	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
1677	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
1678	RT_NOREF(pVmcsInfo);
1679	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
1680	AssertRC(rc);
1681	}
1682
1683
1684	/**
1685	* Exports the guest APIC TPR state into the VMCS.
1686	*
1687	* @param pVCpu The cross context virtual CPU structure.
1688	* @param pVmxTransient The VMX-transient structure.
1689	*
1690	* @remarks No-long-jump zone!!!
1691	*/
1692	static void vmxHCExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1693	{
1694	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
1695	{
1696	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
1697
1698	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1699	if (!pVmxTransient->fIsNestedGuest)
1700	{
1701	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1702	&& PDMApicIsEnabled(pVCpu))
1703	{
1704	/*
1705	* Setup TPR shadowing.
1706	*/
1707	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
1708	{
1709	bool fPendingIntr = false;
1710	uint8_t u8Tpr = 0;
1711	uint8_t u8PendingIntr = 0;
1712	int rc = PDMApicGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
1713	AssertRC(rc);
1714
1715	/*
1716	* If there are interrupts pending but masked by the TPR, instruct VT-x to
1717	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
1718	* priority of the pending interrupt so we can deliver the interrupt. If there
1719	* are no interrupts pending, set threshold to 0 to not cause any
1720	* TPR-below-threshold VM-exits.
1721	*/
1722	uint32_t u32TprThreshold = 0;
1723	if (fPendingIntr)
1724	{
1725	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
1726	(which is the Task-Priority Class). */
1727	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
1728	const uint8_t u8TprPriority = u8Tpr >> 4;
1729	if (u8PendingPriority <= u8TprPriority)
1730	u32TprThreshold = u8PendingPriority;
1731	}
1732
1733	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
1734	}
1735	}
1736	}
1737	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
1738	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
1739	}
1740	}
1741
1742
1743	/**
1744	* Gets the guest interruptibility-state and updates related internal eflags
1745	* inhibition state.
1746	*
1747	* @returns Guest's interruptibility-state.
1748	* @param pVCpu The cross context virtual CPU structure.
1749	*
1750	* @remarks No-long-jump zone!!!
1751	*/
1752	static uint32_t vmxHCGetGuestIntrStateWithUpdate(PVMCPUCC pVCpu)
1753	{
1754	uint32_t fIntrState;
1755
1756	/*
1757	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
1758	*/
1759	if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))
1760	fIntrState = 0;
1761	else
1762	{
1763	/* If inhibition is active, RIP should've been imported from the VMCS already. */
1764	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
1765
1766	if (CPUMIsInInterruptShadowAfterSs(&pVCpu->cpum.GstCtx))
1767	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
1768	else
1769	{
1770	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
1771
1772	/* Block-by-STI must not be set when interrupts are disabled. */
1773	AssertStmt(pVCpu->cpum.GstCtx.eflags.Bits.u1IF, fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
1774	}
1775	}
1776
1777	/*
1778	* Check if we should inhibit NMI delivery.
1779	*/
1780	if (!CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
1781	{ /* likely */ }
1782	else
1783	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
1784
1785	/*
1786	* Validate.
1787	*/
1788	/* We don't support block-by-SMI yet.*/
1789	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
1790
1791	return fIntrState;
1792	}
1793
1794
1795	/**
1796	* Exports the exception intercepts required for guest execution in the VMCS.
1797	*
1798	* @param pVCpu The cross context virtual CPU structure.
1799	* @param pVmxTransient The VMX-transient structure.
1800	*
1801	* @remarks No-long-jump zone!!!
1802	*/
1803	static void vmxHCExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1804	{
1805	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
1806	{
1807	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
1808	if ( !pVmxTransient->fIsNestedGuest
1809	&& VCPU_2_VMXSTATE(pVCpu).fGIMTrapXcptUD)
1810	vmxHCAddXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
1811	else
1812	vmxHCRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
1813
1814	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
1815	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
1816	}
1817	}
1818
1819
1820	/**
1821	* Exports the guest's RIP into the guest-state area in the VMCS.
1822	*
1823	* @param pVCpu The cross context virtual CPU structure.
1824	*
1825	* @remarks No-long-jump zone!!!
1826	*/
1827	static void vmxHCExportGuestRip(PVMCPUCC pVCpu)
1828	{
1829	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RIP)
1830	{
1831	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
1832
1833	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
1834	AssertRC(rc);
1835
1836	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RIP);
1837	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
1838	}
1839	}
1840
1841
1842	/**
1843	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
1844	*
1845	* @param pVCpu The cross context virtual CPU structure.
1846	* @param pVmxTransient The VMX-transient structure.
1847	*
1848	* @remarks No-long-jump zone!!!
1849	*/
1850	static void vmxHCExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1851	{
1852	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
1853	{
1854	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
1855
1856	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits
1857	of RFLAGS are reserved (MBZ). We use bits 63:24 for internal purposes, so no need
1858	to assert this, the CPUMX86EFLAGS/CPUMX86RFLAGS union masks these off for us.
1859	Use 32-bit VMWRITE. */
1860	uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
1861	Assert((fEFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK);
1862	AssertMsg(!(fEFlags & ~(X86_EFL_LIVE_MASK \| X86_EFL_RA1_MASK)), ("%#x\n", fEFlags));
1863
1864	#ifndef IN_NEM_DARWIN
1865	/*
1866	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
1867	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
1868	* can run the real-mode guest code under Virtual 8086 mode.
1869	*/
1870	PVMXVMCSINFOSHARED pVmcsInfo = pVmxTransient->pVmcsInfo->pShared;
1871	if (pVmcsInfo->RealMode.fRealOnV86Active)
1872	{
1873	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
1874	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
1875	Assert(!pVmxTransient->fIsNestedGuest);
1876	pVmcsInfo->RealMode.Eflags.u32 = fEFlags; /* Save the original eflags of the real-mode guest. */
1877	fEFlags \|= X86_EFL_VM; /* Set the Virtual 8086 mode bit. */
1878	fEFlags &= ~X86_EFL_IOPL; /* Change IOPL to 0, otherwise certain instructions won't fault. */
1879	}
1880	#else
1881	RT_NOREF(pVmxTransient);
1882	#endif
1883
1884	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, fEFlags);
1885	AssertRC(rc);
1886
1887	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
1888	Log4Func(("eflags=%#RX32\n", fEFlags));
1889	}
1890	}
1891
1892
1893	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1894	/**
1895	* Copies the nested-guest VMCS to the shadow VMCS.
1896	*
1897	* @returns VBox status code.
1898	* @param pVCpu The cross context virtual CPU structure.
1899	* @param pVmcsInfo The VMCS info. object.
1900	*
1901	* @remarks No-long-jump zone!!!
1902	*/
1903	static int vmxHCCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1904	{
1905	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
1906	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
1907
1908	/*
1909	* Disable interrupts so we don't get preempted while the shadow VMCS is the
1910	* current VMCS, as we may try saving guest lazy MSRs.
1911	*
1912	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
1913	* calling the import VMCS code which is currently performing the guest MSR reads
1914	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
1915	* and the rest of the VMX leave session machinery.
1916	*/
1917	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1918
1919	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
1920	if (RT_SUCCESS(rc))
1921	{
1922	/*
1923	* Copy all guest read/write VMCS fields.
1924	*
1925	* We don't check for VMWRITE failures here for performance reasons and
1926	* because they are not expected to fail, barring irrecoverable conditions
1927	* like hardware errors.
1928	*/
1929	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
1930	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1931	{
1932	uint64_t u64Val;
1933	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
1934	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
1935	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
1936	}
1937
1938	/*
1939	* If the host CPU supports writing all VMCS fields, copy the guest read-only
1940	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
1941	*/
1942	if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
1943	{
1944	uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
1945	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
1946	{
1947	uint64_t u64Val;
1948	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsRoFields[i];
1949	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
1950	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
1951	}
1952	}
1953
1954	rc = vmxHCClearShadowVmcs(pVmcsInfo);
1955	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
1956	}
1957
1958	ASMSetFlags(fEFlags);
1959	return rc;
1960	}
1961
1962
1963	/**
1964	* Copies the shadow VMCS to the nested-guest VMCS.
1965	*
1966	* @returns VBox status code.
1967	* @param pVCpu The cross context virtual CPU structure.
1968	* @param pVmcsInfo The VMCS info. object.
1969	*
1970	* @remarks Called with interrupts disabled.
1971	*/
1972	static int vmxHCCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1973	{
1974	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1975	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
1976	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
1977
1978	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
1979	if (RT_SUCCESS(rc))
1980	{
1981	/*
1982	* Copy guest read/write fields from the shadow VMCS.
1983	* Guest read-only fields cannot be modified, so no need to copy them.
1984	*
1985	* We don't check for VMREAD failures here for performance reasons and
1986	* because they are not expected to fail, barring irrecoverable conditions
1987	* like hardware errors.
1988	*/
1989	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
1990	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1991	{
1992	uint64_t u64Val;
1993	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
1994	VMX_VMCS_READ_64(pVCpu, uVmcsField, &u64Val);
1995	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
1996	}
1997
1998	rc = vmxHCClearShadowVmcs(pVmcsInfo);
1999	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
2000	}
2001	return rc;
2002	}
2003
2004
2005	/**
2006	* Enables VMCS shadowing for the given VMCS info. object.
2007	*
2008	* @param pVCpu The cross context virtual CPU structure.
2009	* @param pVmcsInfo The VMCS info. object.
2010	*
2011	* @remarks No-long-jump zone!!!
2012	*/
2013	static void vmxHCEnableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2014	{
2015	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
2016	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
2017	{
2018	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
2019	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
2020	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
2021	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
2022	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
2023	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
2024	Log4Func(("Enabled\n"));
2025	}
2026	}
2027
2028
2029	/**
2030	* Disables VMCS shadowing for the given VMCS info. object.
2031	*
2032	* @param pVCpu The cross context virtual CPU structure.
2033	* @param pVmcsInfo The VMCS info. object.
2034	*
2035	* @remarks No-long-jump zone!!!
2036	*/
2037	static void vmxHCDisableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2038	{
2039	/*
2040	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
2041	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
2042	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
2043	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
2044	*
2045	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
2046	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
2047	*/
2048	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
2049	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
2050	{
2051	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
2052	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
2053	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
2054	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
2055	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
2056	Log4Func(("Disabled\n"));
2057	}
2058	}
2059	#endif
2060
2061
2062	/**
2063	* Exports the guest CR0 control register into the guest-state area in the VMCS.
2064	*
2065	* The guest FPU state is always pre-loaded hence we don't need to bother about
2066	* sharing FPU related CR0 bits between the guest and host.
2067	*
2068	* @returns VBox status code.
2069	* @param pVCpu The cross context virtual CPU structure.
2070	* @param pVmxTransient The VMX-transient structure.
2071	*
2072	* @remarks No-long-jump zone!!!
2073	*/
2074	static int vmxHCExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2075	{
2076	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR0)
2077	{
2078	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2079	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2080
2081	uint64_t fSetCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed0;
2082	uint64_t const fZapCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed1;
2083	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2084	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
2085	else
2086	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2087
2088	if (!pVmxTransient->fIsNestedGuest)
2089	{
2090	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
2091	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
2092	uint64_t const u64ShadowCr0 = u64GuestCr0;
2093	Assert(!RT_HI_U32(u64GuestCr0));
2094
2095	/*
2096	* Setup VT-x's view of the guest CR0.
2097	*/
2098	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
2099	if (VM_IS_VMX_NESTED_PAGING(pVM))
2100	{
2101	#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
2102	if (CPUMIsGuestPagingEnabled(pVCpu))
2103	{
2104	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
2105	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
2106	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
2107	}
2108	else
2109	{
2110	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
2111	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
2112	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
2113	}
2114
2115	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2116	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2117	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
2118	#endif
2119	}
2120	else
2121	{
2122	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2123	u64GuestCr0 \|= X86_CR0_WP;
2124	}
2125
2126	/*
2127	* Guest FPU bits.
2128	*
2129	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
2130	* using CR0.TS.
2131	*
2132	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
2133	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2134	*/
2135	u64GuestCr0 \|= X86_CR0_NE;
2136
2137	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
2138	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
2139
2140	/*
2141	* Update exception intercepts.
2142	*/
2143	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
2144	#ifndef IN_NEM_DARWIN
2145	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2146	{
2147	Assert(PDMVmmDevHeapIsEnabled(pVM));
2148	Assert(pVM->hm.s.vmx.pRealModeTSS);
2149	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2150	}
2151	else
2152	#endif
2153	{
2154	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
2155	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2156	if (fInterceptMF)
2157	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
2158	}
2159
2160	/* Additional intercepts for debugging, define these yourself explicitly. */
2161	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2162	uXcptBitmap \|= 0
2163	\| RT_BIT(X86_XCPT_BP)
2164	\| RT_BIT(X86_XCPT_DE)
2165	\| RT_BIT(X86_XCPT_NM)
2166	\| RT_BIT(X86_XCPT_TS)
2167	\| RT_BIT(X86_XCPT_UD)
2168	\| RT_BIT(X86_XCPT_NP)
2169	\| RT_BIT(X86_XCPT_SS)
2170	\| RT_BIT(X86_XCPT_GP)
2171	\| RT_BIT(X86_XCPT_PF)
2172	\| RT_BIT(X86_XCPT_MF)
2173	;
2174	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2175	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
2176	#endif
2177	if (VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv)
2178	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
2179	if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE)
2180	uXcptBitmap \|= RT_BIT(X86_XCPT_DE);
2181	Assert(VM_IS_VMX_NESTED_PAGING(pVM) \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
2182
2183	/* Apply the hardware specified CR0 fixed bits and enable caching. */
2184	u64GuestCr0 \|= fSetCr0;
2185	u64GuestCr0 &= fZapCr0;
2186	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
2187
2188	Assert(!RT_HI_U32(u64GuestCr0));
2189	Assert(u64GuestCr0 & X86_CR0_NE);
2190
2191	/* Commit the CR0 and related fields to the guest VMCS. */
2192	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
2193	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
2194	if (uProcCtls != pVmcsInfo->u32ProcCtls)
2195	{
2196	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
2197	AssertRC(rc);
2198	}
2199	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
2200	{
2201	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2202	AssertRC(rc);
2203	}
2204
2205	/* Update our caches. */
2206	pVmcsInfo->u32ProcCtls = uProcCtls;
2207	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2208
2209	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
2210	}
2211	else
2212	{
2213	/*
2214	* With nested-guests, we may have extended the guest/host mask here since we
2215	* merged in the outer guest's mask. Thus, the merged mask can include more bits
2216	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
2217	* originally supplied. We must copy those bits from the nested-guest CR0 into
2218	* the nested-guest CR0 read-shadow.
2219	*/
2220	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
2221	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
2222	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
2223
2224	/* Apply the hardware specified CR0 fixed bits and enable caching. */
2225	u64GuestCr0 \|= fSetCr0;
2226	u64GuestCr0 &= fZapCr0;
2227	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
2228
2229	Assert(!RT_HI_U32(u64GuestCr0));
2230	Assert(u64GuestCr0 & X86_CR0_NE);
2231
2232	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
2233	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
2234	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
2235
2236	Log4Func(("cr0=%#RX64 shadow=%#RX64 vmcs_read_shw=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0,
2237	pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0ReadShadow.u, fSetCr0, fZapCr0));
2238	}
2239
2240	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR0);
2241	}
2242
2243	return VINF_SUCCESS;
2244	}
2245
2246
2247	/**
2248	* Exports the guest control registers (CR3, CR4) into the guest-state area
2249	* in the VMCS.
2250	*
2251	* @returns VBox strict status code.
2252	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
2253	* without unrestricted guest access and the VMMDev is not presently
2254	* mapped (e.g. EFI32).
2255	*
2256	* @param pVCpu The cross context virtual CPU structure.
2257	* @param pVmxTransient The VMX-transient structure.
2258	*
2259	* @remarks No-long-jump zone!!!
2260	*/
2261	static VBOXSTRICTRC vmxHCExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2262	{
2263	int rc = VINF_SUCCESS;
2264	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2265
2266	/*
2267	* Guest CR2.
2268	* It's always loaded in the assembler code. Nothing to do here.
2269	*/
2270
2271	/*
2272	* Guest CR3.
2273	*/
2274	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR3)
2275	{
2276	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
2277
2278	if (VM_IS_VMX_NESTED_PAGING(pVM))
2279	{
2280	#ifndef IN_NEM_DARWIN
2281	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2282	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2283
2284	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2285	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
2286	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2287	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
2288
2289	/* VMX_EPT_MEMTYPE_WB support is already checked in vmxHCSetupTaggedTlb(). */
2290	pVmcsInfo->HCPhysEPTP \|= RT_BF_MAKE(VMX_BF_EPTP_MEMTYPE, VMX_EPTP_MEMTYPE_WB)
2291	\| RT_BF_MAKE(VMX_BF_EPTP_PAGE_WALK_LENGTH, VMX_EPTP_PAGE_WALK_LENGTH_4);
2292
2293	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2294	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2295	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
2296	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
2297	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
2298	\|\| (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY),
2299	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
2300
2301	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
2302	AssertRC(rc);
2303	#endif
2304
2305	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2306	uint64_t u64GuestCr3 = pCtx->cr3;
2307	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
2308	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2309	{
2310	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2311	if (CPUMIsGuestInPAEModeEx(pCtx))
2312	{
2313	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, pCtx->aPaePdpes[0].u); AssertRC(rc);
2314	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, pCtx->aPaePdpes[1].u); AssertRC(rc);
2315	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, pCtx->aPaePdpes[2].u); AssertRC(rc);
2316	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, pCtx->aPaePdpes[3].u); AssertRC(rc);
2317	}
2318
2319	/*
2320	* The guest's view of its CR3 is unblemished with nested paging when the
2321	* guest is using paging or we have unrestricted guest execution to handle
2322	* the guest when it's not using paging.
2323	*/
2324	}
2325	#ifndef IN_NEM_DARWIN
2326	else
2327	{
2328	/*
2329	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
2330	* thinks it accesses physical memory directly, we use our identity-mapped
2331	* page table to map guest-linear to guest-physical addresses. EPT takes care
2332	* of translating it to host-physical addresses.
2333	*/
2334	RTGCPHYS GCPhys;
2335	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2336
2337	/* We obtain it here every time as the guest could have relocated this PCI region. */
2338	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2339	if (RT_SUCCESS(rc))
2340	{ /* likely */ }
2341	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
2342	{
2343	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
2344	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
2345	}
2346	else
2347	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
2348
2349	u64GuestCr3 = GCPhys;
2350	}
2351	#endif
2352
2353	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
2354	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, u64GuestCr3);
2355	AssertRC(rc);
2356	}
2357	else
2358	{
2359	Assert(!pVmxTransient->fIsNestedGuest);
2360	/* Non-nested paging case, just use the hypervisor's CR3. */
2361	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
2362
2363	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
2364	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
2365	AssertRC(rc);
2366	}
2367
2368	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR3);
2369	}
2370
2371	/*
2372	* Guest CR4.
2373	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
2374	*/
2375	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR4)
2376	{
2377	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2378	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2379
2380	uint64_t const fSetCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed0;
2381	uint64_t const fZapCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed1;
2382
2383	/*
2384	* With nested-guests, we may have extended the guest/host mask here (since we
2385	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
2386	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
2387	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
2388	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
2389	*/
2390	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
2391	uint64_t u64GuestCr4 = pCtx->cr4;
2392	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
2393	? pCtx->cr4
2394	: CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
2395	Assert(!RT_HI_U32(u64GuestCr4));
2396
2397	#ifndef IN_NEM_DARWIN
2398	/*
2399	* Setup VT-x's view of the guest CR4.
2400	*
2401	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
2402	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
2403	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2404	*
2405	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2406	*/
2407	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2408	{
2409	Assert(pVM->hm.s.vmx.pRealModeTSS);
2410	Assert(PDMVmmDevHeapIsEnabled(pVM));
2411	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
2412	}
2413	#endif
2414
2415	if (VM_IS_VMX_NESTED_PAGING(pVM))
2416	{
2417	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2418	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2419	{
2420	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2421	u64GuestCr4 \|= X86_CR4_PSE;
2422	/* Our identity mapping is a 32-bit page directory. */
2423	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
2424	}
2425	/* else use guest CR4.*/
2426	}
2427	else
2428	{
2429	Assert(!pVmxTransient->fIsNestedGuest);
2430
2431	/*
2432	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2433	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2434	*/
2435	switch (VCPU_2_VMXSTATE(pVCpu).enmShadowMode)
2436	{
2437	case PGMMODE_REAL: /* Real-mode. */
2438	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2439	case PGMMODE_32_BIT: /* 32-bit paging. */
2440	{
2441	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
2442	break;
2443	}
2444
2445	case PGMMODE_PAE: /* PAE paging. */
2446	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2447	{
2448	u64GuestCr4 \|= X86_CR4_PAE;
2449	break;
2450	}
2451
2452	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2453	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2454	{
2455	#ifdef VBOX_WITH_64_BITS_GUESTS
2456	/* For our assumption in vmxHCShouldSwapEferMsr. */
2457	Assert(u64GuestCr4 & X86_CR4_PAE);
2458	break;
2459	#endif
2460	}
2461	default:
2462	AssertFailed();
2463	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2464	}
2465	}
2466
2467	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
2468	u64GuestCr4 \|= fSetCr4;
2469	u64GuestCr4 &= fZapCr4;
2470
2471	Assert(!RT_HI_U32(u64GuestCr4));
2472	Assert(u64GuestCr4 & X86_CR4_VMXE);
2473
2474	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
2475	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
2476	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
2477
2478	#ifndef IN_NEM_DARWIN
2479	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
2480	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
2481	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
2482	{
2483	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
2484	hmR0VmxUpdateStartVmFunction(pVCpu);
2485	}
2486	#endif
2487
2488	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR4);
2489
2490	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
2491	}
2492	return rc;
2493	}
2494
2495
2496	#ifdef VBOX_STRICT
2497	/**
2498	* Strict function to validate segment registers.
2499	*
2500	* @param pVCpu The cross context virtual CPU structure.
2501	* @param pVmcsInfo The VMCS info. object.
2502	*
2503	* @remarks Will import guest CR0 on strict builds during validation of
2504	* segments.
2505	*/
2506	static void vmxHCValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2507	{
2508	/*
2509	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
2510	*
2511	* The reason we check for attribute value 0 in this function and not just the unusable bit is
2512	* because vmxHCExportGuestSegReg() only updates the VMCS' copy of the value with the
2513	* unusable bit and doesn't change the guest-context value.
2514	*/
2515	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2516	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2517	vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
2518	if ( !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
2519	&& ( !CPUMIsGuestInRealModeEx(pCtx)
2520	&& !CPUMIsGuestInV86ModeEx(pCtx)))
2521	{
2522	/* Protected mode checks */
2523	/* CS */
2524	Assert(pCtx->cs.Attr.n.u1Present);
2525	Assert(!(pCtx->cs.Attr.u & 0xf00));
2526	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
2527	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
2528	\|\| !(pCtx->cs.Attr.n.u1Granularity));
2529	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
2530	\|\| (pCtx->cs.Attr.n.u1Granularity));
2531	/* CS cannot be loaded with NULL in protected mode. */
2532	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
2533	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
2534	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
2535	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
2536	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
2537	else
2538	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
2539	/* SS */
2540	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
2541	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
2542	if ( !(pCtx->cr0 & X86_CR0_PE)
2543	\|\| pCtx->cs.Attr.n.u4Type == 3)
2544	{
2545	Assert(!pCtx->ss.Attr.n.u2Dpl);
2546	}
2547	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
2548	{
2549	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
2550	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
2551	Assert(pCtx->ss.Attr.n.u1Present);
2552	Assert(!(pCtx->ss.Attr.u & 0xf00));
2553	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
2554	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
2555	\|\| !(pCtx->ss.Attr.n.u1Granularity));
2556	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
2557	\|\| (pCtx->ss.Attr.n.u1Granularity));
2558	}
2559	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSegReg(). */
2560	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
2561	{
2562	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2563	Assert(pCtx->ds.Attr.n.u1Present);
2564	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
2565	Assert(!(pCtx->ds.Attr.u & 0xf00));
2566	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
2567	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
2568	\|\| !(pCtx->ds.Attr.n.u1Granularity));
2569	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
2570	\|\| (pCtx->ds.Attr.n.u1Granularity));
2571	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2572	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
2573	}
2574	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
2575	{
2576	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2577	Assert(pCtx->es.Attr.n.u1Present);
2578	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
2579	Assert(!(pCtx->es.Attr.u & 0xf00));
2580	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
2581	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
2582	\|\| !(pCtx->es.Attr.n.u1Granularity));
2583	Assert( !(pCtx->es.u32Limit & 0xfff00000)
2584	\|\| (pCtx->es.Attr.n.u1Granularity));
2585	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2586	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
2587	}
2588	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
2589	{
2590	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2591	Assert(pCtx->fs.Attr.n.u1Present);
2592	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
2593	Assert(!(pCtx->fs.Attr.u & 0xf00));
2594	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
2595	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
2596	\|\| !(pCtx->fs.Attr.n.u1Granularity));
2597	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
2598	\|\| (pCtx->fs.Attr.n.u1Granularity));
2599	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2600	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
2601	}
2602	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
2603	{
2604	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2605	Assert(pCtx->gs.Attr.n.u1Present);
2606	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
2607	Assert(!(pCtx->gs.Attr.u & 0xf00));
2608	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
2609	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
2610	\|\| !(pCtx->gs.Attr.n.u1Granularity));
2611	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
2612	\|\| (pCtx->gs.Attr.n.u1Granularity));
2613	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2614	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
2615	}
2616	/* 64-bit capable CPUs. */
2617	Assert(!RT_HI_U32(pCtx->cs.u64Base));
2618	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
2619	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
2620	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
2621	}
2622	else if ( CPUMIsGuestInV86ModeEx(pCtx)
2623	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
2624	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
2625	{
2626	/* Real and v86 mode checks. */
2627	/* vmxHCExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
2628	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
2629	#ifndef IN_NEM_DARWIN
2630	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2631	{
2632	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
2633	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
2634	}
2635	else
2636	#endif
2637	{
2638	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
2639	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
2640	}
2641
2642	/* CS */
2643	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
2644	Assert(pCtx->cs.u32Limit == 0xffff);
2645	AssertMsg(u32CSAttr == 0xf3, ("cs=%#x %#x ", pCtx->cs.Sel, u32CSAttr));
2646	/* SS */
2647	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
2648	Assert(pCtx->ss.u32Limit == 0xffff);
2649	Assert(u32SSAttr == 0xf3);
2650	/* DS */
2651	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
2652	Assert(pCtx->ds.u32Limit == 0xffff);
2653	Assert(u32DSAttr == 0xf3);
2654	/* ES */
2655	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
2656	Assert(pCtx->es.u32Limit == 0xffff);
2657	Assert(u32ESAttr == 0xf3);
2658	/* FS */
2659	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
2660	Assert(pCtx->fs.u32Limit == 0xffff);
2661	Assert(u32FSAttr == 0xf3);
2662	/* GS */
2663	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
2664	Assert(pCtx->gs.u32Limit == 0xffff);
2665	Assert(u32GSAttr == 0xf3);
2666	/* 64-bit capable CPUs. */
2667	Assert(!RT_HI_U32(pCtx->cs.u64Base));
2668	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
2669	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
2670	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
2671	}
2672	}
2673	#endif /* VBOX_STRICT */
2674
2675
2676	/**
2677	* Exports a guest segment register into the guest-state area in the VMCS.
2678	*
2679	* @returns VBox status code.
2680	* @param pVCpu The cross context virtual CPU structure.
2681	* @param pVmcsInfo The VMCS info. object.
2682	* @param iSegReg The segment register number (X86_SREG_XXX).
2683	* @param pSelReg Pointer to the segment selector.
2684	*
2685	* @remarks No-long-jump zone!!!
2686	*/
2687	static int vmxHCExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
2688	{
2689	Assert(iSegReg < X86_SREG_COUNT);
2690
2691	uint32_t u32Access = pSelReg->Attr.u;
2692	#ifndef IN_NEM_DARWIN
2693	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2694	#endif
2695	{
2696	/*
2697	* The way to differentiate between whether this is really a null selector or was just
2698	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
2699	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
2700	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
2701	* NULL selectors loaded in protected-mode have their attribute as 0.
2702	*/
2703	if (u32Access)
2704	{ }
2705	else
2706	u32Access = X86DESCATTR_UNUSABLE;
2707	}
2708	#ifndef IN_NEM_DARWIN
2709	else
2710	{
2711	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
2712	u32Access = 0xf3;
2713	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2714	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2715	RT_NOREF_PV(pVCpu);
2716	}
2717	#else
2718	RT_NOREF(pVmcsInfo);
2719	#endif
2720
2721	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
2722	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
2723	("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));
2724
2725	/*
2726	* Commit it to the VMCS.
2727	*/
2728	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), pSelReg->Sel); AssertRC(rc);
2729	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), pSelReg->u32Limit); AssertRC(rc);
2730	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), pSelReg->u64Base); AssertRC(rc);
2731	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access); AssertRC(rc);
2732	return VINF_SUCCESS;
2733	}
2734
2735
2736	/**
2737	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
2738	* area in the VMCS.
2739	*
2740	* @returns VBox status code.
2741	* @param pVCpu The cross context virtual CPU structure.
2742	* @param pVmxTransient The VMX-transient structure.
2743	*
2744	* @remarks Will import guest CR0 on strict builds during validation of
2745	* segments.
2746	* @remarks No-long-jump zone!!!
2747	*/
2748	static int vmxHCExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2749	{
2750	int rc = VERR_INTERNAL_ERROR_5;
2751	#ifndef IN_NEM_DARWIN
2752	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2753	#endif
2754	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2755	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2756	#ifndef IN_NEM_DARWIN
2757	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
2758	#endif
2759
2760	/*
2761	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
2762	*/
2763	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
2764	{
2765	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CS)
2766	{
2767	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
2768	#ifndef IN_NEM_DARWIN
2769	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2770	pVmcsInfoShared->RealMode.AttrCS.u = pCtx->cs.Attr.u;
2771	#endif
2772	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
2773	AssertRC(rc);
2774	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CS);
2775	}
2776
2777	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SS)
2778	{
2779	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
2780	#ifndef IN_NEM_DARWIN
2781	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2782	pVmcsInfoShared->RealMode.AttrSS.u = pCtx->ss.Attr.u;
2783	#endif
2784	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
2785	AssertRC(rc);
2786	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SS);
2787	}
2788
2789	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_DS)
2790	{
2791	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
2792	#ifndef IN_NEM_DARWIN
2793	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2794	pVmcsInfoShared->RealMode.AttrDS.u = pCtx->ds.Attr.u;
2795	#endif
2796	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
2797	AssertRC(rc);
2798	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_DS);
2799	}
2800
2801	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_ES)
2802	{
2803	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
2804	#ifndef IN_NEM_DARWIN
2805	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2806	pVmcsInfoShared->RealMode.AttrES.u = pCtx->es.Attr.u;
2807	#endif
2808	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
2809	AssertRC(rc);
2810	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_ES);
2811	}
2812
2813	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_FS)
2814	{
2815	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
2816	#ifndef IN_NEM_DARWIN
2817	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2818	pVmcsInfoShared->RealMode.AttrFS.u = pCtx->fs.Attr.u;
2819	#endif
2820	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
2821	AssertRC(rc);
2822	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_FS);
2823	}
2824
2825	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GS)
2826	{
2827	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
2828	#ifndef IN_NEM_DARWIN
2829	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2830	pVmcsInfoShared->RealMode.AttrGS.u = pCtx->gs.Attr.u;
2831	#endif
2832	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
2833	AssertRC(rc);
2834	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GS);
2835	}
2836
2837	#ifdef VBOX_STRICT
2838	vmxHCValidateSegmentRegs(pVCpu, pVmcsInfo);
2839	#endif
2840	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
2841	pCtx->cs.Attr.u));
2842	}
2843
2844	/*
2845	* Guest TR.
2846	*/
2847	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_TR)
2848	{
2849	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
2850
2851	/*
2852	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
2853	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
2854	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
2855	*/
2856	uint16_t u16Sel;
2857	uint32_t u32Limit;
2858	uint64_t u64Base;
2859	uint32_t u32AccessRights;
2860	#ifndef IN_NEM_DARWIN
2861	if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
2862	#endif
2863	{
2864	u16Sel = pCtx->tr.Sel;
2865	u32Limit = pCtx->tr.u32Limit;
2866	u64Base = pCtx->tr.u64Base;
2867	u32AccessRights = pCtx->tr.Attr.u;
2868	}
2869	#ifndef IN_NEM_DARWIN
2870	else
2871	{
2872	Assert(!pVmxTransient->fIsNestedGuest);
2873	Assert(pVM->hm.s.vmx.pRealModeTSS);
2874	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
2875
2876	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
2877	RTGCPHYS GCPhys;
2878	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
2879	AssertRCReturn(rc, rc);
2880
2881	X86DESCATTR DescAttr;
2882	DescAttr.u = 0;
2883	DescAttr.n.u1Present = 1;
2884	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2885
2886	u16Sel = 0;
2887	u32Limit = HM_VTX_TSS_SIZE;
2888	u64Base = GCPhys;
2889	u32AccessRights = DescAttr.u;
2890	}
2891	#endif
2892
2893	/* Validate. */
2894	Assert(!(u16Sel & RT_BIT(2)));
2895	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
2896	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
2897	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
2898	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
2899	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
2900	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
2901	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
2902	Assert( (u32Limit & 0xfff) == 0xfff
2903	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
2904	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
2905	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
2906
2907	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
2908	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
2909	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
2910	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
2911
2912	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_TR);
2913	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
2914	}
2915
2916	/*
2917	* Guest GDTR.
2918	*/
2919	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GDTR)
2920	{
2921	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
2922
2923	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
2924	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
2925
2926	/* Validate. */
2927	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
2928
2929	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
2930	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
2931	}
2932
2933	/*
2934	* Guest LDTR.
2935	*/
2936	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_LDTR)
2937	{
2938	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
2939
2940	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
2941	uint32_t u32Access;
2942	if ( !pVmxTransient->fIsNestedGuest
2943	&& !pCtx->ldtr.Attr.u)
2944	u32Access = X86DESCATTR_UNUSABLE;
2945	else
2946	u32Access = pCtx->ldtr.Attr.u;
2947
2948	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
2949	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
2950	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
2951	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
2952
2953	/* Validate. */
2954	if (!(u32Access & X86DESCATTR_UNUSABLE))
2955	{
2956	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
2957	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
2958	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
2959	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
2960	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
2961	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
2962	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
2963	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
2964	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
2965	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
2966	}
2967
2968	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
2969	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
2970	}
2971
2972	/*
2973	* Guest IDTR.
2974	*/
2975	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_IDTR)
2976	{
2977	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
2978
2979	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
2980	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
2981
2982	/* Validate. */
2983	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
2984
2985	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
2986	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
2987	}
2988
2989	return VINF_SUCCESS;
2990	}
2991
2992
2993	/**
2994	* Gets the IEM exception flags for the specified vector and IDT vectoring /
2995	* VM-exit interruption info type.
2996	*
2997	* @returns The IEM exception flags.
2998	* @param uVector The event vector.
2999	* @param uVmxEventType The VMX event type.
3000	*
3001	* @remarks This function currently only constructs flags required for
3002	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
3003	* and CR2 aspects of an exception are not included).
3004	*/
3005	static uint32_t vmxHCGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
3006	{
3007	uint32_t fIemXcptFlags;
3008	switch (uVmxEventType)
3009	{
3010	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
3011	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
3012	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
3013	break;
3014
3015	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
3016	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
3017	break;
3018
3019	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
3020	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
3021	break;
3022
3023	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
3024	{
3025	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
3026	if (uVector == X86_XCPT_BP)
3027	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
3028	else if (uVector == X86_XCPT_OF)
3029	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
3030	else
3031	{
3032	fIemXcptFlags = 0;
3033	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
3034	}
3035	break;
3036	}
3037
3038	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
3039	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
3040	break;
3041
3042	default:
3043	fIemXcptFlags = 0;
3044	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
3045	break;
3046	}
3047	return fIemXcptFlags;
3048	}
3049
3050
3051	/**
3052	* Sets an event as a pending event to be injected into the guest.
3053	*
3054	* @param pVCpu The cross context virtual CPU structure.
3055	* @param u32IntInfo The VM-entry interruption-information field.
3056	* @param cbInstr The VM-entry instruction length in bytes (for
3057	* software interrupts, exceptions and privileged
3058	* software exceptions).
3059	* @param u32ErrCode The VM-entry exception error code.
3060	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3061	* page-fault.
3062	*/
3063	DECLINLINE(void) vmxHCSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
3064	RTGCUINTPTR GCPtrFaultAddress)
3065	{
3066	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
3067	VCPU_2_VMXSTATE(pVCpu).Event.fPending = true;
3068	VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo = u32IntInfo;
3069	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode = u32ErrCode;
3070	VCPU_2_VMXSTATE(pVCpu).Event.cbInstr = cbInstr;
3071	VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress = GCPtrFaultAddress;
3072	}
3073
3074
3075	/**
3076	* Sets an external interrupt as pending-for-injection into the VM.
3077	*
3078	* @param pVCpu The cross context virtual CPU structure.
3079	* @param u8Interrupt The external interrupt vector.
3080	*/
3081	DECLINLINE(void) vmxHCSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
3082	{
3083	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
3084	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
3085	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3086	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3087	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3088	Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
3089	}
3090
3091
3092	/**
3093	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
3094	*
3095	* @param pVCpu The cross context virtual CPU structure.
3096	*/
3097	DECLINLINE(void) vmxHCSetPendingXcptNmi(PVMCPUCC pVCpu)
3098	{
3099	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
3100	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
3101	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3102	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3103	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3104	Log4Func(("NMI pending injection\n"));
3105	}
3106
3107
3108	/**
3109	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
3110	*
3111	* @param pVCpu The cross context virtual CPU structure.
3112	*/
3113	DECLINLINE(void) vmxHCSetPendingXcptDF(PVMCPUCC pVCpu)
3114	{
3115	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
3116	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3117	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3118	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3119	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3120	}
3121
3122
3123	/**
3124	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3125	*
3126	* @param pVCpu The cross context virtual CPU structure.
3127	*/
3128	DECLINLINE(void) vmxHCSetPendingXcptUD(PVMCPUCC pVCpu)
3129	{
3130	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
3131	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3132	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3133	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3134	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3135	}
3136
3137
3138	/**
3139	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3140	*
3141	* @param pVCpu The cross context virtual CPU structure.
3142	*/
3143	DECLINLINE(void) vmxHCSetPendingXcptDB(PVMCPUCC pVCpu)
3144	{
3145	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
3146	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3147	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3148	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3149	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3150	}
3151
3152
3153	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3154	/**
3155	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
3156	*
3157	* @param pVCpu The cross context virtual CPU structure.
3158	* @param u32ErrCode The error code for the general-protection exception.
3159	*/
3160	DECLINLINE(void) vmxHCSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
3161	{
3162	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
3163	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3164	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3165	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3166	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
3167	}
3168
3169
3170	/**
3171	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
3172	*
3173	* @param pVCpu The cross context virtual CPU structure.
3174	* @param u32ErrCode The error code for the stack exception.
3175	*/
3176	DECLINLINE(void) vmxHCSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
3177	{
3178	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
3179	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3180	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3181	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3182	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
3183	}
3184	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3185
3186
3187	/**
3188	* Fixes up attributes for the specified segment register.
3189	*
3190	* @param pVCpu The cross context virtual CPU structure.
3191	* @param pSelReg The segment register that needs fixing.
3192	* @param pszRegName The register name (for logging and assertions).
3193	*/
3194	static void vmxHCFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
3195	{
3196	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
3197
3198	/*
3199	* If VT-x marks the segment as unusable, most other bits remain undefined:
3200	* - For CS the L, D and G bits have meaning.
3201	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
3202	* - For the remaining data segments no bits are defined.
3203	*
3204	* The present bit and the unusable bit has been observed to be set at the
3205	* same time (the selector was supposed to be invalid as we started executing
3206	* a V8086 interrupt in ring-0).
3207	*
3208	* What should be important for the rest of the VBox code, is that the P bit is
3209	* cleared. Some of the other VBox code recognizes the unusable bit, but
3210	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
3211	* safe side here, we'll strip off P and other bits we don't care about. If
3212	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
3213	*
3214	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
3215	*/
3216	#ifdef VBOX_STRICT
3217	uint32_t const uAttr = pSelReg->Attr.u;
3218	#endif
3219
3220	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
3221	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
3222	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
3223
3224	#ifdef VBOX_STRICT
3225	# ifndef IN_NEM_DARWIN
3226	VMMRZCallRing3Disable(pVCpu);
3227	# endif
3228	Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
3229	# ifdef DEBUG_bird
3230	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
3231	("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
3232	pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
3233	# endif
3234	# ifndef IN_NEM_DARWIN
3235	VMMRZCallRing3Enable(pVCpu);
3236	# endif
3237	NOREF(uAttr);
3238	#endif
3239	RT_NOREF2(pVCpu, pszRegName);
3240	}
3241
3242
3243	/**
3244	* Imports a guest segment register from the current VMCS into the guest-CPU
3245	* context.
3246	*
3247	* @param pVCpu The cross context virtual CPU structure.
3248	* @tparam a_iSegReg The segment register number (X86_SREG_XXX).
3249	*
3250	* @remarks Called with interrupts and/or preemption disabled.
3251	*/
3252	template<uint32_t const a_iSegReg>
3253	DECLINLINE(void) vmxHCImportGuestSegReg(PVMCPUCC pVCpu)
3254	{
3255	AssertCompile(a_iSegReg < X86_SREG_COUNT);
3256	/* Check that the macros we depend upon here and in the export parenter function works: */
3257	#define MY_SEG_VMCS_FIELD(a_FieldPrefix, a_FieldSuff) \
3258	( a_iSegReg == X86_SREG_ES ? a_FieldPrefix ## ES ## a_FieldSuff \
3259	: a_iSegReg == X86_SREG_CS ? a_FieldPrefix ## CS ## a_FieldSuff \
3260	: a_iSegReg == X86_SREG_SS ? a_FieldPrefix ## SS ## a_FieldSuff \
3261	: a_iSegReg == X86_SREG_DS ? a_FieldPrefix ## DS ## a_FieldSuff \
3262	: a_iSegReg == X86_SREG_FS ? a_FieldPrefix ## FS ## a_FieldSuff \
3263	: a_iSegReg == X86_SREG_GS ? a_FieldPrefix ## GS ## a_FieldSuff : 0)
3264	AssertCompile(VMX_VMCS_GUEST_SEG_BASE(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS_GUEST_,_BASE));
3265	AssertCompile(VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS16_GUEST_,_SEL));
3266	AssertCompile(VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_LIMIT));
3267	AssertCompile(VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_ACCESS_RIGHTS));
3268
3269	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[a_iSegReg];
3270
3271	uint16_t u16Sel;
3272	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg), &u16Sel); AssertRC(rc);
3273	pSelReg->Sel = u16Sel;
3274	pSelReg->ValidSel = u16Sel;
3275
3276	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg), &pSelReg->u32Limit); AssertRC(rc);
3277	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(a_iSegReg), &pSelReg->u64Base); AssertRC(rc);
3278
3279	uint32_t u32Attr;
3280	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg), &u32Attr); AssertRC(rc);
3281	pSelReg->Attr.u = u32Attr;
3282	if (u32Attr & X86DESCATTR_UNUSABLE)
3283	vmxHCFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + a_iSegReg * 3);
3284
3285	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
3286	}
3287
3288
3289	/**
3290	* Imports the guest LDTR from the VMCS into the guest-CPU context.
3291	*
3292	* @param pVCpu The cross context virtual CPU structure.
3293	*
3294	* @remarks Called with interrupts and/or preemption disabled.
3295	*/
3296	DECL_FORCE_INLINE(void) vmxHCImportGuestLdtr(PVMCPUCC pVCpu)
3297	{
3298	uint16_t u16Sel;
3299	uint64_t u64Base;
3300	uint32_t u32Limit, u32Attr;
3301	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
3302	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
3303	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
3304	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
3305
3306	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
3307	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
3308	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
3309	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
3310	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
3311	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
3312	if (u32Attr & X86DESCATTR_UNUSABLE)
3313	vmxHCFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
3314	}
3315
3316
3317	/**
3318	* Imports the guest TR from the VMCS into the guest-CPU context.
3319	*
3320	* @param pVCpu The cross context virtual CPU structure.
3321	*
3322	* @remarks Called with interrupts and/or preemption disabled.
3323	*/
3324	DECL_FORCE_INLINE(void) vmxHCImportGuestTr(PVMCPUCC pVCpu)
3325	{
3326	uint16_t u16Sel;
3327	uint64_t u64Base;
3328	uint32_t u32Limit, u32Attr;
3329	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
3330	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
3331	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
3332	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
3333
3334	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
3335	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
3336	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
3337	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
3338	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
3339	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
3340	/* TR is the only selector that can never be unusable. */
3341	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
3342	}
3343
3344
3345	/**
3346	* Core: Imports the guest RIP from the VMCS into the guest-CPU context.
3347	*
3348	* @returns The RIP value.
3349	* @param pVCpu The cross context virtual CPU structure.
3350	*
3351	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3352	* @remarks Do -not- call this function directly!
3353	*/
3354	DECL_FORCE_INLINE(uint64_t) vmxHCImportGuestCoreRip(PVMCPUCC pVCpu)
3355	{
3356	uint64_t u64Val;
3357	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
3358	AssertRC(rc);
3359
3360	pVCpu->cpum.GstCtx.rip = u64Val;
3361
3362	return u64Val;
3363	}
3364
3365
3366	/**
3367	* Imports the guest RIP from the VMCS into the guest-CPU context.
3368	*
3369	* @param pVCpu The cross context virtual CPU structure.
3370	*
3371	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3372	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3373	* instead!!!
3374	*/
3375	DECL_FORCE_INLINE(void) vmxHCImportGuestRip(PVMCPUCC pVCpu)
3376	{
3377	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP)
3378	{
3379	EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false);
3380	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
3381	}
3382	}
3383
3384
3385	/**
3386	* Core: Imports the guest RFLAGS from the VMCS into the guest-CPU context.
3387	*
3388	* @param pVCpu The cross context virtual CPU structure.
3389	* @param pVmcsInfo The VMCS info. object.
3390	*
3391	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3392	* @remarks Do -not- call this function directly!
3393	*/
3394	DECL_FORCE_INLINE(void) vmxHCImportGuestCoreRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3395	{
3396	uint64_t fRFlags;
3397	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &fRFlags);
3398	AssertRC(rc);
3399
3400	Assert((fRFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK);
3401	Assert((fRFlags & ~(uint64_t)(X86_EFL_1 \| X86_EFL_LIVE_MASK)) == 0);
3402
3403	pVCpu->cpum.GstCtx.rflags.u = fRFlags;
3404	#ifndef IN_NEM_DARWIN
3405	PCVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3406	if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
3407	{ /* mostly likely */ }
3408	else
3409	{
3410	pVCpu->cpum.GstCtx.eflags.Bits.u1VM = 0;
3411	pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL = pVmcsInfoShared->RealMode.Eflags.Bits.u2IOPL;
3412	}
3413	#else
3414	RT_NOREF(pVmcsInfo);
3415	#endif
3416	}
3417
3418
3419	/**
3420	* Imports the guest RFLAGS from the VMCS into the guest-CPU context.
3421	*
3422	* @param pVCpu The cross context virtual CPU structure.
3423	* @param pVmcsInfo The VMCS info. object.
3424	*
3425	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3426	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3427	* instead!!!
3428	*/
3429	DECL_FORCE_INLINE(void) vmxHCImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3430	{
3431	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RFLAGS)
3432	{
3433	vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo);
3434	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
3435	}
3436	}
3437
3438
3439	#ifndef IN_NEM_DARWIN
3440	/**
3441	* Imports the guest TSX AUX and certain other MSRs from the VMCS into the guest-CPU
3442	* context.
3443	*
3444	* The other MSRs are in the VM-exit MSR-store.
3445	*
3446	* @returns VBox status code.
3447	* @param pVCpu The cross context virtual CPU structure.
3448	* @param pVmcsInfo The VMCS info. object.
3449	* @param fEFlags Saved EFLAGS for restoring the interrupt flag (in case of
3450	* unexpected errors). Ignored in NEM/darwin context.
3451	*/
3452	DECL_FORCE_INLINE(int) vmxHCImportGuestTscAuxAndOtherMsrs(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t fEFlags)
3453	{
3454	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3455	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
3456	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
3457	Assert(pMsrs);
3458	Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
3459	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
3460	for (uint32_t i = 0; i < cMsrs; i++)
3461	{
3462	uint32_t const idMsr = pMsrs[i].u32Msr;
3463	switch (idMsr)
3464	{
3465	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
3466	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
3467	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
3468	default:
3469	{
3470	uint32_t idxLbrMsr;
3471	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3472	if (VM_IS_VMX_LBR(pVM))
3473	{
3474	if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
3475	{
3476	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
3477	pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
3478	break;
3479	}
3480	if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
3481	{
3482	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
3483	pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
3484	break;
3485	}
3486	if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
3487	{
3488	pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
3489	break;
3490	}
3491	/* Fallthru (no break) */
3492	}
3493	pVCpu->cpum.GstCtx.fExtrn = 0;
3494	VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
3495	ASMSetFlags(fEFlags);
3496	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
3497	return VERR_HM_UNEXPECTED_LD_ST_MSR;
3498	}
3499	}
3500	}
3501	return VINF_SUCCESS;
3502	}
3503	#endif /* !IN_NEM_DARWIN */
3504
3505
3506	/**
3507	* Imports the guest CR0 from the VMCS into the guest-CPU context.
3508	*
3509	* @param pVCpu The cross context virtual CPU structure.
3510	* @param pVmcsInfo The VMCS info. object.
3511	*/
3512	DECL_FORCE_INLINE(void) vmxHCImportGuestCr0(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3513	{
3514	uint64_t u64Cr0;
3515	uint64_t u64Shadow;
3516	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
3517	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
3518	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
3519	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3520	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
3521	#else
3522	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
3523	{
3524	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3525	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
3526	}
3527	else
3528	{
3529	/*
3530	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
3531	* the nested-guest using hardware-assisted VMX. Accordingly we need to
3532	* re-construct CR0. See @bugref{9180#c95} for details.
3533	*/
3534	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
3535	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3536	u64Cr0 = (u64Cr0 & ~(pVmcsInfoGst->u64Cr0Mask & pVmcsNstGst->u64Cr0Mask.u))
3537	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
3538	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
3539	Assert(u64Cr0 & X86_CR0_NE);
3540	}
3541	#endif
3542
3543	#ifndef IN_NEM_DARWIN
3544	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
3545	#endif
3546	CPUMSetGuestCR0(pVCpu, u64Cr0);
3547	#ifndef IN_NEM_DARWIN
3548	VMMRZCallRing3Enable(pVCpu);
3549	#endif
3550	}
3551
3552
3553	/**
3554	* Imports the guest CR3 from the VMCS into the guest-CPU context.
3555	*
3556	* @param pVCpu The cross context virtual CPU structure.
3557	*/
3558	DECL_FORCE_INLINE(void) vmxHCImportGuestCr3(PVMCPUCC pVCpu)
3559	{
3560	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3561	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3562
3563	/* CR0.PG bit changes are always intercepted, so it's up to date. */
3564	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
3565	\|\| ( VM_IS_VMX_NESTED_PAGING(pVM)
3566	&& CPUMIsGuestPagingEnabledEx(pCtx)))
3567	{
3568	uint64_t u64Cr3;
3569	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
3570	if (pCtx->cr3 != u64Cr3)
3571	{
3572	pCtx->cr3 = u64Cr3;
3573	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
3574	}
3575
3576	/*
3577	* If the guest is in PAE mode, sync back the PDPE's into the guest state.
3578	* CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
3579	*/
3580	if (CPUMIsGuestInPAEModeEx(pCtx))
3581	{
3582	X86PDPE aPaePdpes[4];
3583	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc);
3584	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc);
3585	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc);
3586	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc);
3587	if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes)))
3588	{
3589	memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
3590	/* PGM now updates PAE PDPTEs while updating CR3. */
3591	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
3592	}
3593	}
3594	}
3595	}
3596
3597
3598	/**
3599	* Imports the guest CR4 from the VMCS into the guest-CPU context.
3600	*
3601	* @param pVCpu The cross context virtual CPU structure.
3602	* @param pVmcsInfo The VMCS info. object.
3603	*/
3604	DECL_FORCE_INLINE(void) vmxHCImportGuestCr4(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3605	{
3606	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3607	uint64_t u64Cr4;
3608	uint64_t u64Shadow;
3609	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
3610	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
3611	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
3612	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3613	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
3614	#else
3615	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
3616	{
3617	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3618	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
3619	}
3620	else
3621	{
3622	/*
3623	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
3624	* the nested-guest using hardware-assisted VMX. Accordingly we need to
3625	* re-construct CR4. See @bugref{9180#c95} for details.
3626	*/
3627	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
3628	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3629	u64Cr4 = (u64Cr4 & ~(pVmcsInfo->u64Cr4Mask & pVmcsNstGst->u64Cr4Mask.u))
3630	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
3631	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
3632	Assert(u64Cr4 & X86_CR4_VMXE);
3633	}
3634	#endif
3635	pCtx->cr4 = u64Cr4;
3636	}
3637
3638
3639	/**
3640	* Worker for vmxHCImportGuestIntrState that handles the case where any of the
3641	* relevant VMX_VMCS32_GUEST_INT_STATE bits are set.
3642	*/
3643	DECL_NO_INLINE(static,void) vmxHCImportGuestIntrStateSlow(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t fGstIntState)
3644	{
3645	/*
3646	* We must import RIP here to set our EM interrupt-inhibited state.
3647	* We also import RFLAGS as our code that evaluates pending interrupts
3648	* before VM-entry requires it.
3649	*/
3650	vmxHCImportGuestRip(pVCpu);
3651	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
3652
3653	CPUMUpdateInterruptShadowSsStiEx(&pVCpu->cpum.GstCtx,
3654	RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
3655	RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
3656	pVCpu->cpum.GstCtx.rip);
3657	CPUMUpdateInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx, RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
3658	}
3659
3660
3661	/**
3662	* Imports the guest interruptibility-state from the VMCS into the guest-CPU
3663	* context.
3664	*
3665	* @note May import RIP and RFLAGS if interrupt or NMI are blocked.
3666	*
3667	* @param pVCpu The cross context virtual CPU structure.
3668	* @param pVmcsInfo The VMCS info. object.
3669	*
3670	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
3671	* do not log!
3672	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3673	* instead!!!
3674	*/
3675	DECLINLINE(void) vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3676	{
3677	uint32_t u32Val;
3678	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
3679	Assert((u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
3680	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
3681	if (!u32Val)
3682	{
3683	CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
3684	CPUMClearInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
3685	}
3686	else
3687	vmxHCImportGuestIntrStateSlow(pVCpu, pVmcsInfo, u32Val);
3688	}
3689
3690
3691	/**
3692	* Worker for VMXR0ImportStateOnDemand.
3693	*
3694	* @returns VBox status code.
3695	* @param pVCpu The cross context virtual CPU structure.
3696	* @param pVmcsInfo The VMCS info. object.
3697	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
3698	*/
3699	static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
3700	{
3701	int rc = VINF_SUCCESS;
3702	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3703	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3704	uint32_t u32Val;
3705
3706	/*
3707	* Note! This is hack to workaround a mysterious BSOD observed with release builds
3708	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
3709	* neither are other host platforms.
3710	*
3711	* Committing this temporarily as it prevents BSOD.
3712	*
3713	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
3714	*/
3715	#ifdef RT_OS_WINDOWS
3716	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
3717	return VERR_HM_IPE_1;
3718	#endif
3719
3720	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3721
3722	#ifndef IN_NEM_DARWIN
3723	/*
3724	* We disable interrupts to make the updating of the state and in particular
3725	* the fExtrn modification atomic wrt to preemption hooks.
3726	*/
3727	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
3728	#endif
3729
3730	fWhat &= pCtx->fExtrn;
3731	if (fWhat)
3732	{
3733	do
3734	{
3735	if (fWhat & CPUMCTX_EXTRN_RIP)
3736	vmxHCImportGuestRip(pVCpu);
3737
3738	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
3739	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
3740
3741	/* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */
3742	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
3743	vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
3744
3745	if (fWhat & CPUMCTX_EXTRN_RSP)
3746	{
3747	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pCtx->rsp);
3748	AssertRC(rc);
3749	}
3750
3751	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
3752	{
3753	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3754	#ifndef IN_NEM_DARWIN
3755	bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
3756	#else
3757	bool const fRealOnV86Active = false; /* HV supports only unrestricted guest execution. */
3758	#endif
3759	if (fWhat & CPUMCTX_EXTRN_CS)
3760	{
3761	vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
3762	vmxHCImportGuestRip(pVCpu); /** @todo WTF? */
3763	if (fRealOnV86Active)
3764	pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
3765	EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
3766	}
3767	if (fWhat & CPUMCTX_EXTRN_SS)
3768	{
3769	vmxHCImportGuestSegReg<X86_SREG_SS>(pVCpu);
3770	if (fRealOnV86Active)
3771	pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
3772	}
3773	if (fWhat & CPUMCTX_EXTRN_DS)
3774	{
3775	vmxHCImportGuestSegReg<X86_SREG_DS>(pVCpu);
3776	if (fRealOnV86Active)
3777	pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
3778	}
3779	if (fWhat & CPUMCTX_EXTRN_ES)
3780	{
3781	vmxHCImportGuestSegReg<X86_SREG_ES>(pVCpu);
3782	if (fRealOnV86Active)
3783	pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
3784	}
3785	if (fWhat & CPUMCTX_EXTRN_FS)
3786	{
3787	vmxHCImportGuestSegReg<X86_SREG_FS>(pVCpu);
3788	if (fRealOnV86Active)
3789	pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
3790	}
3791	if (fWhat & CPUMCTX_EXTRN_GS)
3792	{
3793	vmxHCImportGuestSegReg<X86_SREG_GS>(pVCpu);
3794	if (fRealOnV86Active)
3795	pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
3796	}
3797	}
3798
3799	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
3800	{
3801	if (fWhat & CPUMCTX_EXTRN_LDTR)
3802	vmxHCImportGuestLdtr(pVCpu);
3803
3804	if (fWhat & CPUMCTX_EXTRN_GDTR)
3805	{
3806	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
3807	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
3808	pCtx->gdtr.cbGdt = u32Val;
3809	}
3810
3811	/* Guest IDTR. */
3812	if (fWhat & CPUMCTX_EXTRN_IDTR)
3813	{
3814	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
3815	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
3816	pCtx->idtr.cbIdt = u32Val;
3817	}
3818
3819	/* Guest TR. */
3820	if (fWhat & CPUMCTX_EXTRN_TR)
3821	{
3822	#ifndef IN_NEM_DARWIN
3823	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
3824	don't need to import that one. */
3825	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
3826	#endif
3827	vmxHCImportGuestTr(pVCpu);
3828	}
3829	}
3830
3831	if (fWhat & CPUMCTX_EXTRN_DR7)
3832	{
3833	#ifndef IN_NEM_DARWIN
3834	if (!pVCpu->hmr0.s.fUsingHyperDR7)
3835	#endif
3836	{
3837	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);
3838	AssertRC(rc);
3839	}
3840	}
3841
3842	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
3843	{
3844	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
3845	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
3846	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
3847	pCtx->SysEnter.cs = u32Val;
3848	}
3849
3850	#ifndef IN_NEM_DARWIN
3851	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
3852	{
3853	if ( pVM->hmr0.s.fAllow64BitGuests
3854	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
3855	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
3856	}
3857
3858	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
3859	{
3860	if ( pVM->hmr0.s.fAllow64BitGuests
3861	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
3862	{
3863	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3864	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
3865	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
3866	}
3867	}
3868
3869	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
3870	{
3871	rc = vmxHCImportGuestTscAuxAndOtherMsrs(pVCpu, pVmcsInfo, fEFlags);
3872	AssertRCReturn(rc, rc);
3873	}
3874	#else
3875	NOREF(pVM);
3876	#endif
3877
3878	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
3879	{
3880	if (fWhat & CPUMCTX_EXTRN_CR0)
3881	vmxHCImportGuestCr0(pVCpu, pVmcsInfo);
3882
3883	if (fWhat & CPUMCTX_EXTRN_CR4)
3884	vmxHCImportGuestCr4(pVCpu, pVmcsInfo);
3885
3886	if (fWhat & CPUMCTX_EXTRN_CR3)
3887	vmxHCImportGuestCr3(pVCpu);
3888	}
3889
3890	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3891	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
3892	{
3893	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
3894	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
3895	{
3896	Assert(CPUMIsGuestInVmxRootMode(pCtx));
3897	rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
3898	if (RT_SUCCESS(rc))
3899	{ /* likely */ }
3900	else
3901	break;
3902	}
3903	}
3904	#endif
3905	} while (0);
3906
3907	if (RT_SUCCESS(rc))
3908	{
3909	/* Update fExtrn. */
3910	pCtx->fExtrn &= ~fWhat;
3911
3912	/* If everything has been imported, clear the HM keeper bit. */
3913	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
3914	{
3915	#ifndef IN_NEM_DARWIN
3916	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
3917	#else
3918	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
3919	#endif
3920	Assert(!pCtx->fExtrn);
3921	}
3922	}
3923	}
3924	#ifndef IN_NEM_DARWIN
3925	else
3926	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
3927
3928	/*
3929	* Restore interrupts.
3930	*/
3931	ASMSetFlags(fEFlags);
3932	#endif
3933
3934	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3935
3936	if (RT_SUCCESS(rc))
3937	{ /* likely */ }
3938	else
3939	return rc;
3940
3941	/*
3942	* Honor any pending CR3 updates.
3943	*
3944	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
3945	* -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
3946	* -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
3947	*
3948	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
3949	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
3950	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
3951	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
3952	*
3953	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
3954	*
3955	* The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
3956	*/
3957	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)
3958	#ifndef IN_NEM_DARWIN
3959	&& VMMRZCallRing3IsEnabled(pVCpu)
3960	#endif
3961	)
3962	{
3963	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
3964	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3965	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
3966	}
3967
3968	return VINF_SUCCESS;
3969	}
3970
3971
3972	/**
3973	* Internal state fetcher, inner version where we fetch all of a_fWhat.
3974	*
3975	* @returns VBox status code.
3976	* @param pVCpu The cross context virtual CPU structure.
3977	* @param pVmcsInfo The VMCS info. object.
3978	* @param fEFlags Saved EFLAGS for restoring the interrupt flag. Ignored
3979	* in NEM/darwin context.
3980	* @tparam a_fWhat What to import, zero or more bits from
3981	* HMVMX_CPUMCTX_EXTRN_ALL.
3982	*/
3983	template<uint64_t const a_fWhat>
3984	static int vmxHCImportGuestStateInner(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t fEFlags)
3985	{
3986	Assert(a_fWhat != 0); /* No AssertCompile as the assertion probably kicks in before the compiler (clang) discards it. */
3987	AssertCompile(!(a_fWhat & ~HMVMX_CPUMCTX_EXTRN_ALL));
3988	Assert( (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == a_fWhat
3989	\|\| (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == (a_fWhat & ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)));
3990
3991	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3992
3993	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3994
3995	/* RIP and RFLAGS may have been imported already by the post exit code
3996	together with the CPUMCTX_EXTRN_INHIBIT_INT/NMI state, so this part
3997	of the code is skipping this part of the code. */
3998	if ( (a_fWhat & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS))
3999	&& pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS))
4000	{
4001	if (a_fWhat & CPUMCTX_EXTRN_RFLAGS)
4002	vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo);
4003
4004	if (a_fWhat & CPUMCTX_EXTRN_RIP)
4005	{
4006	if (!(a_fWhat & CPUMCTX_EXTRN_CS))
4007	EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false);
4008	else
4009	vmxHCImportGuestCoreRip(pVCpu);
4010	}
4011	}
4012
4013	/* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */
4014	if (a_fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
4015	vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
4016
4017	if (a_fWhat & (CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TR))
4018	{
4019	if (a_fWhat & CPUMCTX_EXTRN_CS)
4020	{
4021	vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
4022	/** @todo try get rid of this carp, it smells and is probably never ever
4023	* used: */
4024	if ( !(a_fWhat & CPUMCTX_EXTRN_RIP)
4025	&& (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP))
4026	{
4027	vmxHCImportGuestCoreRip(pVCpu);
4028	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
4029	}
4030	EMHistoryUpdatePC(pVCpu, pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, true /* fFlattened */);
4031	}
4032	if (a_fWhat & CPUMCTX_EXTRN_SS)
4033	vmxHCImportGuestSegReg<X86_SREG_SS>(pVCpu);
4034	if (a_fWhat & CPUMCTX_EXTRN_DS)
4035	vmxHCImportGuestSegReg<X86_SREG_DS>(pVCpu);
4036	if (a_fWhat & CPUMCTX_EXTRN_ES)
4037	vmxHCImportGuestSegReg<X86_SREG_ES>(pVCpu);
4038	if (a_fWhat & CPUMCTX_EXTRN_FS)
4039	vmxHCImportGuestSegReg<X86_SREG_FS>(pVCpu);
4040	if (a_fWhat & CPUMCTX_EXTRN_GS)
4041	vmxHCImportGuestSegReg<X86_SREG_GS>(pVCpu);
4042
4043	/* Guest TR.
4044	Real-mode emulation using virtual-8086 mode has the fake TSS
4045	(pRealModeTSS) in TR, don't need to import that one. */
4046	#ifndef IN_NEM_DARWIN
4047	PVMXVMCSINFOSHARED const pVmcsInfoShared = pVmcsInfo->pShared;
4048	bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
4049	if ((a_fWhat & CPUMCTX_EXTRN_TR) && !fRealOnV86Active)
4050	#else
4051	if (a_fWhat & CPUMCTX_EXTRN_TR)
4052	#endif
4053	vmxHCImportGuestTr(pVCpu);
4054
4055	#ifndef IN_NEM_DARWIN /* NEM/Darwin: HV supports only unrestricted guest execution. */
4056	if (fRealOnV86Active)
4057	{
4058	if (a_fWhat & CPUMCTX_EXTRN_CS)
4059	pVCpu->cpum.GstCtx.cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
4060	if (a_fWhat & CPUMCTX_EXTRN_SS)
4061	pVCpu->cpum.GstCtx.ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
4062	if (a_fWhat & CPUMCTX_EXTRN_DS)
4063	pVCpu->cpum.GstCtx.ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
4064	if (a_fWhat & CPUMCTX_EXTRN_ES)
4065	pVCpu->cpum.GstCtx.es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
4066	if (a_fWhat & CPUMCTX_EXTRN_FS)
4067	pVCpu->cpum.GstCtx.fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
4068	if (a_fWhat & CPUMCTX_EXTRN_GS)
4069	pVCpu->cpum.GstCtx.gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
4070	}
4071	#endif
4072	}
4073
4074	if (a_fWhat & CPUMCTX_EXTRN_RSP)
4075	{
4076	int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pVCpu->cpum.GstCtx.rsp);
4077	AssertRC(rc);
4078	}
4079
4080	if (a_fWhat & CPUMCTX_EXTRN_LDTR)
4081	vmxHCImportGuestLdtr(pVCpu);
4082
4083	if (a_fWhat & CPUMCTX_EXTRN_GDTR)
4084	{
4085	int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pVCpu->cpum.GstCtx.gdtr.pGdt); AssertRC(rc1);
4086	uint32_t u32Val;
4087	int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc2);
4088	pVCpu->cpum.GstCtx.gdtr.cbGdt = (uint16_t)u32Val;
4089	}
4090
4091	/* Guest IDTR. */
4092	if (a_fWhat & CPUMCTX_EXTRN_IDTR)
4093	{
4094	int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pVCpu->cpum.GstCtx.idtr.pIdt); AssertRC(rc1);
4095	uint32_t u32Val;
4096	int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc2);
4097	pVCpu->cpum.GstCtx.idtr.cbIdt = (uint64_t)u32Val;
4098	}
4099
4100	if (a_fWhat & CPUMCTX_EXTRN_DR7)
4101	{
4102	#ifndef IN_NEM_DARWIN
4103	if (!pVCpu->hmr0.s.fUsingHyperDR7)
4104	#endif
4105	{
4106	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pVCpu->cpum.GstCtx.dr[7]);
4107	AssertRC(rc);
4108	}
4109	}
4110
4111	if (a_fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
4112	{
4113	int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pVCpu->cpum.GstCtx.SysEnter.eip); AssertRC(rc1);
4114	int const rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pVCpu->cpum.GstCtx.SysEnter.esp); AssertRC(rc2);
4115	uint32_t u32Val;
4116	int const rc3 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc3);
4117	pVCpu->cpum.GstCtx.SysEnter.cs = u32Val;
4118	}
4119
4120	#ifndef IN_NEM_DARWIN
4121	if (a_fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
4122	{
4123	if ( (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
4124	&& pVM->hmr0.s.fAllow64BitGuests)
4125	pVCpu->cpum.GstCtx.msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
4126	}
4127
4128	if (a_fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
4129	{
4130	if ( (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
4131	&& pVM->hmr0.s.fAllow64BitGuests)
4132	{
4133	pVCpu->cpum.GstCtx.msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
4134	pVCpu->cpum.GstCtx.msrSTAR = ASMRdMsr(MSR_K6_STAR);
4135	pVCpu->cpum.GstCtx.msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
4136	}
4137	}
4138
4139	if (a_fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
4140	{
4141	int const rc1 = vmxHCImportGuestTscAuxAndOtherMsrs(pVCpu, pVmcsInfo, fEFlags);
4142	AssertRCReturn(rc1, rc1);
4143	}
4144	#else
4145	NOREF(pVM);
4146	#endif
4147
4148	if (a_fWhat & CPUMCTX_EXTRN_CR0)
4149	vmxHCImportGuestCr0(pVCpu, pVmcsInfo);
4150
4151	if (a_fWhat & CPUMCTX_EXTRN_CR4)
4152	vmxHCImportGuestCr4(pVCpu, pVmcsInfo);
4153
4154	if (a_fWhat & CPUMCTX_EXTRN_CR3)
4155	vmxHCImportGuestCr3(pVCpu);
4156
4157	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4158	if (a_fWhat & CPUMCTX_EXTRN_HWVIRT)
4159	{
4160	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
4161	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
4162	{
4163	Assert(CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx));
4164	int const rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
4165	AssertRCReturn(rc, rc);
4166	}
4167	}
4168	#endif
4169
4170	/* Update fExtrn. */
4171	pVCpu->cpum.GstCtx.fExtrn &= ~a_fWhat;
4172
4173	/* If everything has been imported, clear the HM keeper bit. */
4174	if (!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
4175	{
4176	#ifndef IN_NEM_DARWIN
4177	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
4178	#else
4179	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
4180	#endif
4181	Assert(!pVCpu->cpum.GstCtx.fExtrn);
4182	}
4183
4184	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
4185
4186	/*
4187	* Honor any pending CR3 updates.
4188	*
4189	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
4190	* -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
4191	* -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
4192	*
4193	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
4194	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
4195	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
4196	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
4197	*
4198	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
4199	*
4200	* The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
4201	*/
4202	#ifndef IN_NEM_DARWIN
4203	if (!(a_fWhat & CPUMCTX_EXTRN_CR3)
4204	? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) \|\| !VMMRZCallRing3IsEnabled(pVCpu))
4205	: !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) \|\| !VMMRZCallRing3IsEnabled(pVCpu) )
4206	return VINF_SUCCESS;
4207	ASMSetFlags(fEFlags);
4208	#else
4209	if (!(a_fWhat & CPUMCTX_EXTRN_CR3)
4210	? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
4211	: !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) )
4212	return VINF_SUCCESS;
4213	RT_NOREF_PV(fEFlags);
4214	#endif
4215
4216	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
4217	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
4218	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
4219	return VINF_SUCCESS;
4220	}
4221
4222
4223	/**
4224	* Internal state fetcher.
4225	*
4226	* @returns VBox status code.
4227	* @param pVCpu The cross context virtual CPU structure.
4228	* @param pVmcsInfo The VMCS info. object.
4229	* @param pszCaller For logging.
4230	* @tparam a_fWhat What needs to be imported, CPUMCTX_EXTRN_XXX.
4231	* @tparam a_fDoneLocal What's ASSUMED to have been retrieved locally
4232	* already. This is ORed together with @a a_fWhat when
4233	* calculating what needs fetching (just for safety).
4234	* @tparam a_fDonePostExit What's ASSUMED to been been retrieved by
4235	* hmR0VmxPostRunGuest()/nemR3DarwinHandleExitCommon()
4236	* already. This is ORed together with @a a_fWhat when
4237	* calculating what needs fetching (just for safety).
4238	*/
4239	template<uint64_t const a_fWhat,
4240	uint64_t const a_fDoneLocal = 0,
4241	uint64_t const a_fDonePostExit = 0
4242	#ifndef IN_NEM_DARWIN
4243	\| CPUMCTX_EXTRN_INHIBIT_INT
4244	\| CPUMCTX_EXTRN_INHIBIT_NMI
4245	# if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
4246	\| HMVMX_CPUMCTX_EXTRN_ALL
4247	# elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
4248	\| CPUMCTX_EXTRN_RFLAGS
4249	# endif
4250	#else /* IN_NEM_DARWIN */
4251	\| CPUMCTX_EXTRN_ALL /** @todo optimize */
4252	#endif /* IN_NEM_DARWIN */
4253	>
4254	DECLINLINE(int) vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, const char *pszCaller)
4255	{
4256	RT_NOREF_PV(pszCaller);
4257	if ((a_fWhat \| a_fDoneLocal \| a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL)
4258	{
4259	#ifndef IN_NEM_DARWIN
4260	/*
4261	* We disable interrupts to make the updating of the state and in particular
4262	* the fExtrn modification atomic wrt to preemption hooks.
4263	*/
4264	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
4265	#else
4266	RTCCUINTREG const fEFlags = 0;
4267	#endif
4268
4269	/*
4270	* We combine all three parameters and take the (probably) inlined optimized
4271	* code path for the new things specified in a_fWhat.
4272	*
4273	* As a tweak to deal with exits that have INHIBIT_INT/NMI active, causing
4274	* vmxHCImportGuestIntrState to automatically fetch both RIP & RFLAGS, we
4275	* also take the streamlined path when both of these are cleared in fExtrn
4276	* already. vmxHCImportGuestStateInner checks fExtrn before fetching. This
4277	* helps with MWAIT and HLT exits that always inhibit IRQs on many platforms.
4278	*/
4279	uint64_t const fWhatToDo = pVCpu->cpum.GstCtx.fExtrn
4280	& ((a_fWhat \| a_fDoneLocal \| a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL);
4281	if (RT_LIKELY( ( fWhatToDo == (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal \| a_fDonePostExit))
4282	\|\| fWhatToDo == ( a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal \| a_fDonePostExit)
4283	& ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)) /* fetch with INHIBIT_INT/NMI */))
4284	&& (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal \| a_fDonePostExit)) != 0 /* just in case */)
4285	{
4286	int const rc = vmxHCImportGuestStateInner< a_fWhat
4287	& HMVMX_CPUMCTX_EXTRN_ALL
4288	& ~(a_fDoneLocal \| a_fDonePostExit)>(pVCpu, pVmcsInfo, fEFlags);
4289	#ifndef IN_NEM_DARWIN
4290	ASMSetFlags(fEFlags);
4291	#endif
4292	return rc;
4293	}
4294
4295	#ifndef IN_NEM_DARWIN
4296	ASMSetFlags(fEFlags);
4297	#endif
4298
4299	/*
4300	* We shouldn't normally get here, but it may happen when executing
4301	* in the debug run-loops. Typically, everything should already have
4302	* been fetched then. Otherwise call the fallback state import function.
4303	*/
4304	if (fWhatToDo == 0)
4305	{ /* hope the cause was the debug loop or something similar */ }
4306	else
4307	{
4308	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestStateFallback);
4309	Log11Func(("a_fWhat=%#RX64/%#RX64/%#RX64 fExtrn=%#RX64 => %#RX64 - Taking inefficient code path from %s!\n",
4310	a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL, a_fDoneLocal & HMVMX_CPUMCTX_EXTRN_ALL,
4311	a_fDonePostExit & HMVMX_CPUMCTX_EXTRN_ALL, pVCpu->cpum.GstCtx.fExtrn, fWhatToDo, pszCaller));
4312	return vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, a_fWhat \| a_fDoneLocal \| a_fDonePostExit);
4313	}
4314	}
4315	return VINF_SUCCESS;
4316	}
4317
4318
4319	/**
4320	* Check per-VM and per-VCPU force flag actions that require us to go back to
4321	* ring-3 for one reason or another.
4322	*
4323	* @returns Strict VBox status code (i.e. informational status codes too)
4324	* @retval VINF_SUCCESS if we don't have any actions that require going back to
4325	* ring-3.
4326	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
4327	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
4328	* interrupts)
4329	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
4330	* all EMTs to be in ring-3.
4331	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
4332	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
4333	* to the EM loop.
4334	*
4335	* @param pVCpu The cross context virtual CPU structure.
4336	* @param fIsNestedGuest Flag whether this is for a for a pending nested guest event.
4337	* @param fStepping Whether we are single-stepping the guest using the
4338	* hypervisor debugger.
4339	*
4340	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
4341	* is no longer in VMX non-root mode.
4342	*/
4343	static VBOXSTRICTRC vmxHCCheckForceFlags(PVMCPUCC pVCpu, bool fIsNestedGuest, bool fStepping)
4344	{
4345	#ifndef IN_NEM_DARWIN
4346	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4347	#endif
4348
4349	/*
4350	* Update pending interrupts into the APIC's IRR.
4351	*/
4352	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
4353	PDMApicUpdatePendingInterrupts(pVCpu);
4354
4355	/*
4356	* Anything pending? Should be more likely than not if we're doing a good job.
4357	*/
4358	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4359	if ( !fStepping
4360	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
4361	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
4362	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
4363	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
4364	return VINF_SUCCESS;
4365
4366	/* Pending PGM C3 sync. */
4367	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
4368	{
4369	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4370	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
4371	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
4372	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4373	if (rcStrict != VINF_SUCCESS)
4374	{
4375	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
4376	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
4377	return rcStrict;
4378	}
4379	}
4380
4381	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
4382	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
4383	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4384	{
4385	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHmToR3FF);
4386	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
4387	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d (fVM=%#RX64 fCpu=%#RX64)\n",
4388	rc, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
4389	return rc;
4390	}
4391
4392	/* Pending VM request packets, such as hardware interrupts. */
4393	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
4394	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
4395	{
4396	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchVmReq);
4397	Log4Func(("Pending VM request forcing us back to ring-3\n"));
4398	return VINF_EM_PENDING_REQUEST;
4399	}
4400
4401	/* Pending PGM pool flushes. */
4402	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4403	{
4404	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchPgmPoolFlush);
4405	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4406	return VINF_PGM_POOL_FLUSH_PENDING;
4407	}
4408
4409	/* Pending DMA requests. */
4410	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4411	{
4412	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchDma);
4413	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4414	return VINF_EM_RAW_TO_R3;
4415	}
4416
4417	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4418	/*
4419	* Pending nested-guest events.
4420	*
4421	* Please note the priority of these events are specified and important.
4422	* See Intel spec. 29.4.3.2 "APIC-Write Emulation".
4423	* See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
4424	*
4425	* Interrupt-window and NMI-window VM-exits for the nested-guest need not be
4426	* handled here. They'll be handled by the hardware while executing the nested-guest
4427	* or by us when we injecting events that are not part of VM-entry of the nested-guest.
4428	*/
4429	if (fIsNestedGuest)
4430	{
4431	/* Pending nested-guest APIC-write (may or may not cause a VM-exit). */
4432	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
4433	{
4434	Log4Func(("Pending nested-guest APIC-write\n"));
4435	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
4436	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
4437	if ( rcStrict == VINF_SUCCESS
4438	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
4439	return rcStrict;
4440	}
4441
4442	/* Pending nested-guest monitor-trap flag (MTF). */
4443	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
4444	{
4445	Log4Func(("Pending nested-guest MTF\n"));
4446	VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
4447	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
4448	return rcStrict;
4449	}
4450
4451	/* Pending nested-guest VMX-preemption timer expired. */
4452	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
4453	{
4454	Log4Func(("Pending nested-guest preempt timer\n"));
4455	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
4456	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
4457	return rcStrict;
4458	}
4459	}
4460	#else
4461	NOREF(fIsNestedGuest);
4462	#endif
4463
4464	return VINF_SUCCESS;
4465	}
4466
4467
4468	/**
4469	* Converts any TRPM trap into a pending HM event. This is typically used when
4470	* entering from ring-3 (not longjmp returns).
4471	*
4472	* @param pVCpu The cross context virtual CPU structure.
4473	*/
4474	static void vmxHCTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
4475	{
4476	Assert(TRPMHasTrap(pVCpu));
4477	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
4478
4479	uint8_t uVector;
4480	TRPMEVENT enmTrpmEvent;
4481	uint32_t uErrCode;
4482	RTGCUINTPTR GCPtrFaultAddress;
4483	uint8_t cbInstr;
4484	bool fIcebp;
4485
4486	uVector = TRPMGetTrapAll(pVCpu, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
4487
4488	uint32_t u32IntInfo;
4489	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
4490	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
4491
4492	int const rc = TRPMResetTrap(pVCpu);
4493	AssertRC(rc);
4494	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
4495	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
4496
4497	vmxHCSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
4498	}
4499
4500
4501	/**
4502	* Converts the pending HM event into a TRPM trap.
4503	*
4504	* @param pVCpu The cross context virtual CPU structure.
4505	*/
4506	static void vmxHCPendingEventToTrpmTrap(PVMCPUCC pVCpu)
4507	{
4508	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
4509
4510	/* If a trap was already pending, we did something wrong! */
4511	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
4512
4513	uint32_t const u32IntInfo = VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo;
4514	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
4515	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
4516
4517	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
4518
4519	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
4520	AssertRC(rc);
4521
4522	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
4523	TRPMSetErrorCode(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode);
4524
4525	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
4526	TRPMSetFaultAddress(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress);
4527	else
4528	{
4529	uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
4530	switch (uVectorType)
4531	{
4532	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
4533	TRPMSetTrapDueToIcebp(pVCpu);
4534	RT_FALL_THRU();
4535	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
4536	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
4537	{
4538	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4539	\|\| ( uVector == X86_XCPT_BP /* INT3 */
4540	\|\| uVector == X86_XCPT_OF /* INTO */
4541	\|\| uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
4542	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
4543	TRPMSetInstrLength(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.cbInstr);
4544	break;
4545	}
4546	}
4547	}
4548
4549	/* We're now done converting the pending event. */
4550	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
4551	}
4552
4553
4554	/**
4555	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
4556	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
4557	*
4558	* @param pVCpu The cross context virtual CPU structure.
4559	* @param pVmcsInfo The VMCS info. object.
4560	*/
4561	static void vmxHCSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4562	{
4563	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
4564	{
4565	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
4566	{
4567	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
4568	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4569	AssertRC(rc);
4570	}
4571	Log4Func(("Enabled interrupt-window exiting\n"));
4572	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
4573	}
4574
4575
4576	/**
4577	* Clears the interrupt-window exiting control in the VMCS.
4578	*
4579	* @param pVCpu The cross context virtual CPU structure.
4580	* @param pVmcsInfo The VMCS info. object.
4581	*/
4582	DECLINLINE(void) vmxHCClearIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4583	{
4584	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
4585	{
4586	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
4587	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4588	AssertRC(rc);
4589	Log4Func(("Disabled interrupt-window exiting\n"));
4590	}
4591	}
4592
4593
4594	/**
4595	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
4596	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
4597	*
4598	* @param pVCpu The cross context virtual CPU structure.
4599	* @param pVmcsInfo The VMCS info. object.
4600	*/
4601	static void vmxHCSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4602	{
4603	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
4604	{
4605	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
4606	{
4607	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
4608	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4609	AssertRC(rc);
4610	Log4Func(("Enabled NMI-window exiting\n"));
4611	}
4612	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
4613	}
4614
4615
4616	/**
4617	* Clears the NMI-window exiting control in the VMCS.
4618	*
4619	* @param pVCpu The cross context virtual CPU structure.
4620	* @param pVmcsInfo The VMCS info. object.
4621	*/
4622	DECLINLINE(void) vmxHCClearNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4623	{
4624	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
4625	{
4626	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
4627	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4628	AssertRC(rc);
4629	Log4Func(("Disabled NMI-window exiting\n"));
4630	}
4631	}
4632
4633
4634	/**
4635	* Injects an event into the guest upon VM-entry by updating the relevant fields
4636	* in the VM-entry area in the VMCS.
4637	*
4638	* @returns Strict VBox status code (i.e. informational status codes too).
4639	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
4640	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
4641	*
4642	* @param pVCpu The cross context virtual CPU structure.
4643	* @param pVmcsInfo The VMCS info object.
4644	* @param fIsNestedGuest Flag whether this is for a for a pending nested guest event.
4645	* @param pEvent The event being injected.
4646	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
4647	* will be updated if necessary. This cannot not be NULL.
4648	* @param fStepping Whether we're single-stepping guest execution and should
4649	* return VINF_EM_DBG_STEPPED if the event is injected
4650	* directly (registers modified by us, not by hardware on
4651	* VM-entry).
4652	*/
4653	static VBOXSTRICTRC vmxHCInjectEventVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, PCHMEVENT pEvent,
4654	bool fStepping, uint32_t *pfIntrState)
4655	{
4656	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
4657	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
4658	Assert(pfIntrState);
4659
4660	#ifdef IN_NEM_DARWIN
4661	RT_NOREF(fIsNestedGuest, fStepping, pfIntrState);
4662	#endif
4663
4664	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4665	uint32_t u32IntInfo = pEvent->u64IntInfo;
4666	uint32_t const u32ErrCode = pEvent->u32ErrCode;
4667	uint32_t const cbInstr = pEvent->cbInstr;
4668	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
4669	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
4670	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
4671
4672	#ifdef VBOX_STRICT
4673	/*
4674	* Validate the error-code-valid bit for hardware exceptions.
4675	* No error codes for exceptions in real-mode.
4676	*
4677	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
4678	*/
4679	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
4680	&& !CPUMIsGuestInRealModeEx(pCtx))
4681	{
4682	switch (uVector)
4683	{
4684	case X86_XCPT_PF:
4685	case X86_XCPT_DF:
4686	case X86_XCPT_TS:
4687	case X86_XCPT_NP:
4688	case X86_XCPT_SS:
4689	case X86_XCPT_GP:
4690	case X86_XCPT_AC:
4691	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
4692	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
4693	RT_FALL_THRU();
4694	default:
4695	break;
4696	}
4697	}
4698
4699	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
4700	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
4701	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
4702	#endif
4703
4704	RT_NOREF(uVector);
4705	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
4706	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
4707	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
4708	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
4709	{
4710	Assert(uVector <= X86_XCPT_LAST);
4711	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI \|\| uVector == X86_XCPT_NMI);
4712	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT \|\| uVector == X86_XCPT_DB);
4713	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedXcpts[uVector]);
4714	}
4715	else
4716	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedIrqs[uVector & MASK_INJECT_IRQ_STAT]);
4717
4718	/*
4719	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
4720	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
4721	* interrupt handler in the (real-mode) guest.
4722	*
4723	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
4724	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
4725	*/
4726	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
4727	{
4728	#ifndef IN_NEM_DARWIN
4729	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest)
4730	#endif
4731	{
4732	/*
4733	* For CPUs with unrestricted guest execution enabled and with the guest
4734	* in real-mode, we must not set the deliver-error-code bit.
4735	*
4736	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
4737	*/
4738	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
4739	}
4740	#ifndef IN_NEM_DARWIN
4741	else
4742	{
4743	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4744	Assert(PDMVmmDevHeapIsEnabled(pVM));
4745	Assert(pVM->hm.s.vmx.pRealModeTSS);
4746	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
4747
4748	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
4749	int rc2 = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
4750	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
4751	AssertRCReturn(rc2, rc2);
4752
4753	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
4754	size_t const cbIdtEntry = sizeof(X86IDTR16);
4755	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
4756	{
4757	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
4758	if (uVector == X86_XCPT_DF)
4759	return VINF_EM_RESET;
4760
4761	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
4762	No error codes for exceptions in real-mode. */
4763	if (uVector == X86_XCPT_GP)
4764	{
4765	static HMEVENT const s_EventXcptDf
4766	= HMEVENT_INIT_ONLY_INT_INFO( RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
4767	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
4768	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4769	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1));
4770	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptDf, fStepping, pfIntrState);
4771	}
4772
4773	/*
4774	* If we're injecting an event with no valid IDT entry, inject a #GP.
4775	* No error codes for exceptions in real-mode.
4776	*
4777	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
4778	*/
4779	static HMEVENT const s_EventXcptGp
4780	= HMEVENT_INIT_ONLY_INT_INFO( RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
4781	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
4782	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4783	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1));
4784	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptGp, fStepping, pfIntrState);
4785	}
4786
4787	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
4788	uint16_t uGuestIp = pCtx->ip;
4789	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
4790	{
4791	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
4792	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
4793	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
4794	}
4795	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
4796	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
4797
4798	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
4799	X86IDTR16 IdtEntry;
4800	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
4801	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
4802	AssertRCReturn(rc2, rc2);
4803
4804	/* Construct the stack frame for the interrupt/exception handler. */
4805	VBOXSTRICTRC rcStrict;
4806	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, (uint16_t)pCtx->eflags.u);
4807	if (rcStrict == VINF_SUCCESS)
4808	{
4809	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
4810	if (rcStrict == VINF_SUCCESS)
4811	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
4812	}
4813
4814	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
4815	if (rcStrict == VINF_SUCCESS)
4816	{
4817	pCtx->eflags.u &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
4818	pCtx->rip = IdtEntry.offSel;
4819	pCtx->cs.Sel = IdtEntry.uSel;
4820	pCtx->cs.ValidSel = IdtEntry.uSel;
4821	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
4822	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
4823	&& uVector == X86_XCPT_PF)
4824	pCtx->cr2 = GCPtrFault;
4825
4826	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
4827	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
4828	\| HM_CHANGED_GUEST_RSP);
4829
4830	/*
4831	* If we delivered a hardware exception (other than an NMI) and if there was
4832	* block-by-STI in effect, we should clear it.
4833	*/
4834	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
4835	{
4836	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
4837	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
4838	Log4Func(("Clearing inhibition due to STI\n"));
4839	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4840	}
4841
4842	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
4843	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
4844
4845	/*
4846	* The event has been truly dispatched to the guest. Mark it as no longer pending so
4847	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
4848	*/
4849	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
4850
4851	/*
4852	* If we eventually support nested-guest execution without unrestricted guest execution,
4853	* we should set fInterceptEvents here.
4854	*/
4855	Assert(!fIsNestedGuest);
4856
4857	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
4858	if (fStepping)
4859	rcStrict = VINF_EM_DBG_STEPPED;
4860	}
4861	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
4862	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
4863	return rcStrict;
4864	}
4865	#else
4866	RT_NOREF(pVmcsInfo);
4867	#endif
4868	}
4869
4870	/*
4871	* Validate.
4872	*/
4873	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
4874	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
4875
4876	/*
4877	* Inject the event into the VMCS.
4878	*/
4879	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
4880	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
4881	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
4882	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
4883	AssertRC(rc);
4884
4885	/*
4886	* Update guest CR2 if this is a page-fault.
4887	*/
4888	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
4889	pCtx->cr2 = GCPtrFault;
4890
4891	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
4892	return VINF_SUCCESS;
4893	}
4894
4895
4896	/**
4897	* Evaluates the event to be delivered to the guest and sets it as the pending
4898	* event.
4899	*
4900	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
4901	* exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
4902	* NOT restore these force-flags.
4903	*
4904	* @returns Strict VBox status code (i.e. informational status codes too).
4905	* @param pVCpu The cross context virtual CPU structure.
4906	* @param pVmcsInfo The VMCS information structure.
4907	* @param pfIntrState Where to store the updated VMX guest-interruptibility
4908	* state.
4909	*/
4910	static VBOXSTRICTRC vmxHCEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t *pfIntrState)
4911	{
4912	Assert(pfIntrState);
4913	Assert(!TRPMHasTrap(pVCpu));
4914
4915	*pfIntrState = vmxHCGetGuestIntrStateWithUpdate(pVCpu);
4916
4917	/*
4918	* Evaluate if a new event needs to be injected.
4919	* An event that's already pending has already performed all necessary checks.
4920	*/
4921	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
4922	&& !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
4923	{
4924	/** @todo SMI. SMIs take priority over NMIs. */
4925
4926	/*
4927	* NMIs.
4928	* NMIs take priority over external interrupts.
4929	*/
4930	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
4931	{
4932	if (!CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
4933	{
4934	/* Finally, inject the NMI and we're done. */
4935	vmxHCSetPendingXcptNmi(pVCpu);
4936	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
4937	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4938	return VINF_SUCCESS;
4939	}
4940	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4941	}
4942	else
4943	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4944
4945	/*
4946	* External interrupts (PIC/APIC).
4947	*/
4948	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
4949	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
4950	{
4951	Assert(!DBGFIsStepping(pVCpu));
4952	int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
4953	AssertRC(rc);
4954
4955	if (pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF)
4956	{
4957	/*
4958	* Once PDMGetInterrupt() returns an interrupt we -must- deliver it.
4959	* We cannot re-request the interrupt from the controller again.
4960	*/
4961	uint8_t u8Interrupt;
4962	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
4963	if (RT_SUCCESS(rc))
4964	vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
4965	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
4966	{
4967	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchTprMaskedIrq);
4968	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4969	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
4970	/*
4971	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
4972	* PDMApicSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
4973	* need to re-set this force-flag here.
4974	*/
4975	}
4976	else
4977	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchGuestIrq);
4978
4979	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
4980	return VINF_SUCCESS;
4981	}
4982	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
4983	}
4984	else
4985	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
4986	}
4987	else
4988	{
4989	/*
4990	* An event is being injected or we are in an interrupt shadow.
4991	* If another event is pending currently, instruct VT-x to cause a VM-exit as
4992	* soon as the guest is ready to accept it.
4993	*/
4994	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
4995	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4996	else
4997	{
4998	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT));
4999	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
5000	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
5001	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
5002	else
5003	{
5004	/* It's possible that interrupt-window exiting is still active, clear it as it's now unnecessary. */
5005	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
5006	}
5007	}
5008	}
5009
5010	return VINF_SUCCESS;
5011	}
5012
5013
5014	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5015	/**
5016	* Evaluates the event to be delivered to the nested-guest and sets it as the
5017	* pending event.
5018	*
5019	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
5020	* exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
5021	* NOT restore these force-flags.
5022	*
5023	* @returns Strict VBox status code (i.e. informational status codes too).
5024	* @param pVCpu The cross context virtual CPU structure.
5025	* @param pVmcsInfo The VMCS information structure.
5026	* @param pfIntrState Where to store the updated VMX guest-interruptibility
5027	* state.
5028	*
5029	* @remarks The guest must be in VMX non-root mode.
5030	*/
5031	static VBOXSTRICTRC vmxHCEvaluatePendingEventNested(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t *pfIntrState)
5032	{
5033	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5034
5035	Assert(pfIntrState);
5036	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
5037	Assert(!TRPMHasTrap(pVCpu));
5038
5039	*pfIntrState = vmxHCGetGuestIntrStateWithUpdate(pVCpu);
5040
5041	/*
5042	* If we are injecting an event, all necessary checks have been performed.
5043	* Any interrupt-window or NMI-window exiting would have been setup by the
5044	* nested-guest while we merged controls.
5045	*/
5046	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
5047	return VINF_SUCCESS;
5048
5049	/*
5050	* An event injected by VMLAUNCH/VMRESUME instruction emulation should've been
5051	* made pending (TRPM to HM event) and would be handled above if we resumed
5052	* execution in HM. If somehow we fell back to emulation after the
5053	* VMLAUNCH/VMRESUME instruction, it would have been handled in iemRaiseXcptOrInt
5054	* (calling iemVmxVmexitEvent). Thus, if we get here the nested-hypervisor's VMX
5055	* intercepts should be active and any events pending here have been generated
5056	* while executing the guest in VMX non-root mode after virtual VM-entry completed.
5057	*/
5058	Assert(CPUMIsGuestVmxInterceptEvents(pCtx));
5059
5060	/*
5061	* Interrupt shadows MAY block NMIs.
5062	* They also blocks external-interrupts and MAY block external-interrupt VM-exits.
5063	*
5064	* See Intel spec. 24.4.2 "Guest Non-Register State".
5065	* See Intel spec. 25.4.1 "Event Blocking".
5066	*/
5067	if (!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
5068	{ /* likely */ }
5069	else
5070	return VINF_SUCCESS;
5071
5072	/** @todo SMI. SMIs take priority over NMIs. */
5073
5074	/*
5075	* NMIs.
5076	* NMIs take priority over interrupts.
5077	*/
5078	if (!CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
5079	{
5080	/*
5081	* Nested-guest NMI-window exiting.
5082	* The NMI-window exit must happen regardless of whether an NMI is pending
5083	* provided virtual-NMI blocking is not in effect.
5084	*
5085	* See Intel spec. 25.2 "Other Causes Of VM Exits".
5086	*/
5087	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
5088	&& !CPUMIsGuestVmxVirtNmiBlocking(pCtx))
5089	{
5090	Assert(CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT));
5091	return IEMExecVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
5092	}
5093
5094	/*
5095	* For a nested-guest, the FF always indicates the outer guest's ability to
5096	* receive an NMI while the guest-interruptibility state bit depends on whether
5097	* the nested-hypervisor is using virtual-NMIs.
5098	*
5099	* It is very important that we also clear the force-flag if we are causing
5100	* an NMI VM-exit as it is the responsibility of the nested-hypervisor to deal
5101	* with re-injecting or discarding the NMI. This fixes the bug that showed up
5102	* with SMP Windows Server 2008 R2 with Hyper-V enabled, see @bugref{10318#c19}.
5103	*/
5104	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5105	{
5106	if (CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
5107	return IEMExecVmxVmexitXcptNmi(pVCpu);
5108	vmxHCSetPendingXcptNmi(pVCpu);
5109	return VINF_SUCCESS;
5110	}
5111	}
5112
5113	/*
5114	* Nested-guest interrupt-window exiting.
5115	*
5116	* We must cause the interrupt-window exit regardless of whether an interrupt is pending
5117	* provided virtual interrupts are enabled.
5118	*
5119	* See Intel spec. 25.2 "Other Causes Of VM Exits".
5120	* See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
5121	*/
5122	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
5123	&& CPUMIsGuestVmxVirtIntrEnabled(pCtx))
5124	{
5125	Assert(CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT));
5126	return IEMExecVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
5127	}
5128
5129	/*
5130	* External interrupts (PIC/APIC).
5131	*
5132	* When "External interrupt exiting" is set the VM-exit happens regardless of RFLAGS.IF.
5133	* When it isn't set, RFLAGS.IF controls delivery of the interrupt as always.
5134	* This fixes a nasty SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued
5135	* by other VM-exits (like a preemption timer), see @bugref{9562#c18}.
5136	*
5137	* NMIs block external interrupts as they are dispatched through the interrupt gate (vector 2)
5138	* which automatically clears EFLAGS.IF. Also it's possible an NMI handler could enable interrupts
5139	* and thus we should not check for NMI inhibition here.
5140	*
5141	* See Intel spec. 25.4.1 "Event Blocking".
5142	* See Intel spec. 6.8.1 "Masking Maskable Hardware Interrupts".
5143	*/
5144	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
5145	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
5146	{
5147	Assert(!DBGFIsStepping(pVCpu));
5148	int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
5149	AssertRC(rc);
5150	if (CPUMIsGuestVmxPhysIntrEnabled(pCtx))
5151	{
5152	/* Nested-guest external interrupt VM-exit. */
5153	if ( CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
5154	&& !CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
5155	{
5156	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
5157	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5158	return rcStrict;
5159	}
5160
5161	/*
5162	* Fetch the external interrupt from the interrupt controller.
5163	* Once PDMGetInterrupt() returns an interrupt we -must- deliver it or pass it to
5164	* the nested-hypervisor. We cannot re-request the interrupt from the controller again.
5165	*/
5166	uint8_t u8Interrupt;
5167	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5168	if (RT_SUCCESS(rc))
5169	{
5170	/* Nested-guest external interrupt VM-exit when the "acknowledge interrupt on exit" is enabled. */
5171	if (CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
5172	{
5173	Assert(CPUMIsGuestVmxExitCtlsSet(pCtx, VMX_EXIT_CTLS_ACK_EXT_INT));
5174	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
5175	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
5176	return rcStrict;
5177	}
5178	vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
5179	return VINF_SUCCESS;
5180	}
5181	}
5182	}
5183	return VINF_SUCCESS;
5184	}
5185	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
5186
5187
5188	/**
5189	* Injects any pending events into the guest if the guest is in a state to
5190	* receive them.
5191	*
5192	* @returns Strict VBox status code (i.e. informational status codes too).
5193	* @param pVCpu The cross context virtual CPU structure.
5194	* @param pVmcsInfo The VMCS information structure.
5195	* @param fIsNestedGuest Flag whether the event injection happens for a nested guest.
5196	* @param fIntrState The VT-x guest-interruptibility state.
5197	* @param fStepping Whether we are single-stepping the guest using the
5198	* hypervisor debugger and should return
5199	* VINF_EM_DBG_STEPPED if the event was dispatched
5200	* directly.
5201	*/
5202	static VBOXSTRICTRC vmxHCInjectPendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest,
5203	uint32_t fIntrState, bool fStepping)
5204	{
5205	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
5206	#ifndef IN_NEM_DARWIN
5207	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5208	#endif
5209
5210	#ifdef VBOX_STRICT
5211	/*
5212	* Verify guest-interruptibility state.
5213	*
5214	* We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
5215	* since injecting an event may modify the interruptibility state and we must thus always
5216	* use fIntrState.
5217	*/
5218	{
5219	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5220	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
5221	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
5222	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5223	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
5224	Assert(!TRPMHasTrap(pVCpu));
5225	NOREF(fBlockMovSS); NOREF(fBlockSti);
5226	}
5227	#endif
5228
5229	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5230	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
5231	{
5232	/*
5233	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
5234	* pending even while injecting an event and in this case, we want a VM-exit as soon as
5235	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
5236	*
5237	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
5238	*/
5239	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo);
5240	#ifdef VBOX_STRICT
5241	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
5242	{
5243	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
5244	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
5245	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
5246	}
5247	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
5248	{
5249	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
5250	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
5251	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
5252	}
5253	#endif
5254	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
5255	uIntType));
5256
5257	/*
5258	* Inject the event and get any changes to the guest-interruptibility state.
5259	*
5260	* The guest-interruptibility state may need to be updated if we inject the event
5261	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
5262	*/
5263	rcStrict = vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &VCPU_2_VMXSTATE(pVCpu).Event, fStepping, &fIntrState);
5264	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
5265
5266	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
5267	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterrupt);
5268	else
5269	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectXcpt);
5270	}
5271
5272	/*
5273	* Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
5274	* is an interrupt shadow (block-by-STI or block-by-MOV SS).
5275	*/
5276	if ( (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5277	&& !fIsNestedGuest)
5278	{
5279	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5280
5281	if (!VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
5282	{
5283	/*
5284	* Set or clear the BS bit depending on whether the trap flag is active or not. We need
5285	* to do both since we clear the BS bit from the VMCS while exiting to ring-3.
5286	*/
5287	Assert(!DBGFIsStepping(pVCpu));
5288	uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_TF);
5289	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
5290	fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
5291	AssertRC(rc);
5292	}
5293	else
5294	{
5295	/*
5296	* We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
5297	* hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
5298	* we take care of this case in vmxHCExportSharedDebugState and also the case if
5299	* we use MTF, so just make sure it's called before executing guest-code.
5300	*/
5301	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
5302	}
5303	}
5304	/* else: for nested-guest currently handling while merging controls. */
5305
5306	/*
5307	* Finally, update the guest-interruptibility state.
5308	*
5309	* This is required for the real-on-v86 software interrupt injection, for
5310	* pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
5311	*/
5312	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5313	AssertRC(rc);
5314
5315	/*
5316	* There's no need to clear the VM-entry interruption-information field here if we're not
5317	* injecting anything. VT-x clears the valid bit on every VM-exit.
5318	*
5319	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5320	*/
5321
5322	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
5323	return rcStrict;
5324	}
5325
5326
5327	/**
5328	* Tries to determine what part of the guest-state VT-x has deemed as invalid
5329	* and update error record fields accordingly.
5330	*
5331	* @returns VMX_IGS_* error codes.
5332	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
5333	* wrong with the guest state.
5334	*
5335	* @param pVCpu The cross context virtual CPU structure.
5336	* @param pVmcsInfo The VMCS info. object.
5337	*
5338	* @remarks This function assumes our cache of the VMCS controls
5339	* are valid, i.e. vmxHCCheckCachedVmcsCtls() succeeded.
5340	*/
5341	static uint32_t vmxHCCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
5342	{
5343	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
5344	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { uError = (err); break; } else do { } while (0)
5345
5346	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5347	uint32_t uError = VMX_IGS_ERROR;
5348	uint32_t u32IntrState = 0;
5349	#ifndef IN_NEM_DARWIN
5350	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5351	bool const fUnrestrictedGuest = VM_IS_VMX_UNRESTRICTED_GUEST(pVM);
5352	#else
5353	bool const fUnrestrictedGuest = true;
5354	#endif
5355	do
5356	{
5357	int rc;
5358
5359	/*
5360	* Guest-interruptibility state.
5361	*
5362	* Read this first so that any check that fails prior to those that actually
5363	* require the guest-interruptibility state would still reflect the correct
5364	* VMCS value and avoids causing further confusion.
5365	*/
5366	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
5367	AssertRC(rc);
5368
5369	uint32_t u32Val;
5370	uint64_t u64Val;
5371
5372	/*
5373	* CR0.
5374	*/
5375	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5376	uint64_t fSetCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 & g_HmMsrs.u.vmx.u64Cr0Fixed1);
5377	uint64_t const fZapCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 \| g_HmMsrs.u.vmx.u64Cr0Fixed1);
5378	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
5379	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
5380	if (fUnrestrictedGuest)
5381	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5382
5383	uint64_t u64GuestCr0;
5384	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64GuestCr0);
5385	AssertRC(rc);
5386	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
5387	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
5388	if ( !fUnrestrictedGuest
5389	&& (u64GuestCr0 & X86_CR0_PG)
5390	&& !(u64GuestCr0 & X86_CR0_PE))
5391	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
5392
5393	/*
5394	* CR4.
5395	*/
5396	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5397	uint64_t const fSetCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 & g_HmMsrs.u.vmx.u64Cr4Fixed1);
5398	uint64_t const fZapCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 \| g_HmMsrs.u.vmx.u64Cr4Fixed1);
5399
5400	uint64_t u64GuestCr4;
5401	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64GuestCr4);
5402	AssertRC(rc);
5403	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
5404	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
5405
5406	/*
5407	* IA32_DEBUGCTL MSR.
5408	*/
5409	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
5410	AssertRC(rc);
5411	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5412	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
5413	{
5414	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
5415	}
5416	uint64_t u64DebugCtlMsr = u64Val;
5417
5418	#ifdef VBOX_STRICT
5419	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
5420	AssertRC(rc);
5421	Assert(u32Val == pVmcsInfo->u32EntryCtls);
5422	#endif
5423	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
5424
5425	/*
5426	* RIP and RFLAGS.
5427	*/
5428	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
5429	AssertRC(rc);
5430	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
5431	if ( !fLongModeGuest
5432	\|\| !pCtx->cs.Attr.n.u1Long)
5433	{
5434	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
5435	}
5436	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
5437	* must be identical if the "IA-32e mode guest" VM-entry
5438	* control is 1 and CS.L is 1. No check applies if the
5439	* CPU supports 64 linear-address bits. */
5440
5441	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
5442	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
5443	AssertRC(rc);
5444	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
5445	VMX_IGS_RFLAGS_RESERVED);
5446	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
5447	uint32_t const u32Eflags = u64Val;
5448
5449	if ( fLongModeGuest
5450	\|\| ( fUnrestrictedGuest
5451	&& !(u64GuestCr0 & X86_CR0_PE)))
5452	{
5453	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
5454	}
5455
5456	uint32_t u32EntryInfo;
5457	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
5458	AssertRC(rc);
5459	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
5460	{
5461	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
5462	}
5463
5464	/*
5465	* 64-bit checks.
5466	*/
5467	if (fLongModeGuest)
5468	{
5469	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
5470	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
5471	}
5472
5473	if ( !fLongModeGuest
5474	&& (u64GuestCr4 & X86_CR4_PCIDE))
5475	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
5476
5477	/** @todo CR3 field must be such that bits 63:52 and bits in the range
5478	* 51:32 beyond the processor's physical-address width are 0. */
5479
5480	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5481	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
5482	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
5483
5484	#ifndef IN_NEM_DARWIN
5485	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
5486	AssertRC(rc);
5487	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
5488
5489	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
5490	AssertRC(rc);
5491	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
5492	#endif
5493
5494	/*
5495	* PERF_GLOBAL MSR.
5496	*/
5497	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
5498	{
5499	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
5500	AssertRC(rc);
5501	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
5502	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
5503	}
5504
5505	/*
5506	* PAT MSR.
5507	*/
5508	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
5509	{
5510	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
5511	AssertRC(rc);
5512	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
5513	for (unsigned i = 0; i < 8; i++)
5514	{
5515	uint8_t u8Val = (u64Val & 0xff);
5516	if ( u8Val > MSR_IA32_PAT_MT_UCD
5517	\|\| u8Val == MSR_IA32_PAT_MT_RSVD_2
5518	\|\| u8Val == MSR_IA32_PAT_MT_RSVD_3)
5519	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
5520	u64Val >>= 8;
5521	}
5522	}
5523
5524	/*
5525	* EFER MSR.
5526	*/
5527	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
5528	{
5529	Assert(g_fHmVmxSupportsVmcsEfer);
5530	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
5531	AssertRC(rc);
5532	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
5533	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
5534	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
5535	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
5536	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
5537	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
5538	* iemVmxVmentryCheckGuestState(). */
5539	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5540	\|\| !(u64GuestCr0 & X86_CR0_PG)
5541	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
5542	VMX_IGS_EFER_LMA_LME_MISMATCH);
5543	}
5544
5545	/*
5546	* Segment registers.
5547	*/
5548	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
5549	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
5550	if (!(u32Eflags & X86_EFL_VM))
5551	{
5552	/* CS */
5553	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
5554	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
5555	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
5556	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5557	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
5558	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
5559	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
5560	/* CS cannot be loaded with NULL in protected mode. */
5561	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
5562	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
5563	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5564	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
5565	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5566	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
5567	else if (fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
5568	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
5569	else
5570	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
5571
5572	/* SS */
5573	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5574	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
5575	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
5576	if ( !(pCtx->cr0 & X86_CR0_PE)
5577	\|\| pCtx->cs.Attr.n.u4Type == 3)
5578	{
5579	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
5580	}
5581
5582	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5583	{
5584	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
5585	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
5586	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
5587	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
5588	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5589	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
5590	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
5591	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
5592	}
5593
5594	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSReg(). */
5595	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5596	{
5597	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
5598	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
5599	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5600	\|\| pCtx->ds.Attr.n.u4Type > 11
5601	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
5602	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
5603	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
5604	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5605	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
5606	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
5607	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
5608	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5609	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
5610	}
5611	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5612	{
5613	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
5614	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
5615	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5616	\|\| pCtx->es.Attr.n.u4Type > 11
5617	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
5618	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
5619	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
5620	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
5621	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
5622	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
5623	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
5624	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5625	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
5626	}
5627	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5628	{
5629	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
5630	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
5631	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5632	\|\| pCtx->fs.Attr.n.u4Type > 11
5633	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
5634	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
5635	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
5636	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5637	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
5638	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
5639	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
5640	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5641	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
5642	}
5643	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5644	{
5645	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
5646	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
5647	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5648	\|\| pCtx->gs.Attr.n.u4Type > 11
5649	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
5650	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
5651	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
5652	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5653	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
5654	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
5655	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
5656	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5657	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
5658	}
5659	/* 64-bit capable CPUs. */
5660	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
5661	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
5662	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
5663	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
5664	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
5665	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
5666	VMX_IGS_LONGMODE_SS_BASE_INVALID);
5667	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
5668	VMX_IGS_LONGMODE_DS_BASE_INVALID);
5669	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
5670	VMX_IGS_LONGMODE_ES_BASE_INVALID);
5671	}
5672	else
5673	{
5674	/* V86 mode checks. */
5675	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5676	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
5677	{
5678	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
5679	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
5680	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5681	}
5682	else
5683	{
5684	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
5685	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
5686	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5687	}
5688
5689	/* CS */
5690	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
5691	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
5692	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
5693	/* SS */
5694	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
5695	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
5696	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
5697	/* DS */
5698	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
5699	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
5700	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
5701	/* ES */
5702	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
5703	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
5704	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
5705	/* FS */
5706	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
5707	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
5708	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
5709	/* GS */
5710	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
5711	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
5712	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
5713	/* 64-bit capable CPUs. */
5714	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
5715	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
5716	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
5717	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
5718	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
5719	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
5720	VMX_IGS_LONGMODE_SS_BASE_INVALID);
5721	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
5722	VMX_IGS_LONGMODE_DS_BASE_INVALID);
5723	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
5724	VMX_IGS_LONGMODE_ES_BASE_INVALID);
5725	}
5726
5727	/*
5728	* TR.
5729	*/
5730	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
5731	/* 64-bit capable CPUs. */
5732	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
5733	if (fLongModeGuest)
5734	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
5735	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
5736	else
5737	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
5738	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
5739	VMX_IGS_TR_ATTR_TYPE_INVALID);
5740	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
5741	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
5742	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
5743	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
5744	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
5745	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
5746	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
5747	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
5748
5749	/*
5750	* GDTR and IDTR (64-bit capable checks).
5751	*/
5752	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
5753	AssertRC(rc);
5754	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
5755
5756	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
5757	AssertRC(rc);
5758	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
5759
5760	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
5761	AssertRC(rc);
5762	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
5763
5764	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
5765	AssertRC(rc);
5766	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
5767
5768	/*
5769	* Guest Non-Register State.
5770	*/
5771	/* Activity State. */
5772	uint32_t u32ActivityState;
5773	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
5774	AssertRC(rc);
5775	HMVMX_CHECK_BREAK( !u32ActivityState
5776	\|\| (u32ActivityState & RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
5777	VMX_IGS_ACTIVITY_STATE_INVALID);
5778	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
5779	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
5780
5781	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
5782	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5783	{
5784	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
5785	}
5786
5787	/** @todo Activity state and injecting interrupts. Left as a todo since we
5788	* currently don't use activity states but ACTIVE. */
5789
5790	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
5791	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
5792
5793	/* Guest interruptibility-state. */
5794	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
5795	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5796	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
5797	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
5798	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
5799	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
5800	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
5801	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
5802	{
5803	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5804	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
5805	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
5806	}
5807	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
5808	{
5809	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
5810	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
5811	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
5812	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
5813	}
5814	/** @todo Assumes the processor is not in SMM. */
5815	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
5816	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
5817	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
5818	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
5819	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
5820	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
5821	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
5822	{
5823	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
5824	}
5825
5826	/* Pending debug exceptions. */
5827	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
5828	AssertRC(rc);
5829	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
5830	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
5831	u32Val = u64Val; /* For pending debug exceptions checks below. */
5832
5833	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5834	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
5835	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
5836	{
5837	if ( (u32Eflags & X86_EFL_TF)
5838	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
5839	{
5840	/* Bit 14 is PendingDebug.BS. */
5841	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
5842	}
5843	if ( !(u32Eflags & X86_EFL_TF)
5844	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
5845	{
5846	/* Bit 14 is PendingDebug.BS. */
5847	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
5848	}
5849	}
5850
5851	#ifndef IN_NEM_DARWIN
5852	/* VMCS link pointer. */
5853	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
5854	AssertRC(rc);
5855	if (u64Val != UINT64_C(0xffffffffffffffff))
5856	{
5857	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
5858	/** @todo Bits beyond the processor's physical-address width MBZ. */
5859	/** @todo SMM checks. */
5860	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
5861	Assert(pVmcsInfo->pvShadowVmcs);
5862	VMXVMCSREVID VmcsRevId;
5863	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
5864	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID),
5865	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
5866	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
5867	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
5868	}
5869
5870	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
5871	* not using nested paging? */
5872	if ( VM_IS_VMX_NESTED_PAGING(pVM)
5873	&& !fLongModeGuest
5874	&& CPUMIsGuestInPAEModeEx(pCtx))
5875	{
5876	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
5877	AssertRC(rc);
5878	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5879
5880	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
5881	AssertRC(rc);
5882	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5883
5884	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
5885	AssertRC(rc);
5886	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5887
5888	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
5889	AssertRC(rc);
5890	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5891	}
5892	#endif
5893
5894	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
5895	if (uError == VMX_IGS_ERROR)
5896	uError = VMX_IGS_REASON_NOT_FOUND;
5897	} while (0);
5898
5899	VCPU_2_VMXSTATE(pVCpu).u32HMError = uError;
5900	VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32GuestIntrState = u32IntrState;
5901	return uError;
5902
5903	#undef HMVMX_ERROR_BREAK
5904	#undef HMVMX_CHECK_BREAK
5905	}
5906
5907
5908	#ifndef HMVMX_USE_FUNCTION_TABLE
5909	/**
5910	* Handles a guest VM-exit from hardware-assisted VMX execution.
5911	*
5912	* @returns Strict VBox status code (i.e. informational status codes too).
5913	* @param pVCpu The cross context virtual CPU structure.
5914	* @param pVmxTransient The VMX-transient structure.
5915	*/
5916	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5917	{
5918	#ifdef DEBUG_ramshankar
5919	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
5920	do { \
5921	if (a_fSave != 0) \
5922	vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); \
5923	VBOXSTRICTRC rcStrict = a_CallExpr; \
5924	if (a_fSave != 0) \
5925	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); \
5926	return rcStrict; \
5927	} while (0)
5928	#else
5929	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
5930	#endif
5931	uint32_t const uExitReason = pVmxTransient->uExitReason;
5932	switch (uExitReason)
5933	{
5934	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, vmxHCExitEptMisconfig(pVCpu, pVmxTransient));
5935	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, vmxHCExitEptViolation(pVCpu, pVmxTransient));
5936	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, vmxHCExitIoInstr(pVCpu, pVmxTransient));
5937	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, vmxHCExitCpuid(pVCpu, pVmxTransient));
5938	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, vmxHCExitRdtsc(pVCpu, pVmxTransient));
5939	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, vmxHCExitRdtscp(pVCpu, pVmxTransient));
5940	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, vmxHCExitApicAccess(pVCpu, pVmxTransient));
5941	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, vmxHCExitXcptOrNmi(pVCpu, pVmxTransient));
5942	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, vmxHCExitMovCRx(pVCpu, pVmxTransient));
5943	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, vmxHCExitExtInt(pVCpu, pVmxTransient));
5944	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, vmxHCExitIntWindow(pVCpu, pVmxTransient));
5945	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient));
5946	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, vmxHCExitMwait(pVCpu, pVmxTransient));
5947	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, vmxHCExitMonitor(pVCpu, pVmxTransient));
5948	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, vmxHCExitTaskSwitch(pVCpu, pVmxTransient));
5949	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, vmxHCExitPreemptTimer(pVCpu, pVmxTransient));
5950	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, vmxHCExitRdmsr(pVCpu, pVmxTransient));
5951	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, vmxHCExitWrmsr(pVCpu, pVmxTransient));
5952	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, vmxHCExitVmcall(pVCpu, pVmxTransient));
5953	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, vmxHCExitMovDRx(pVCpu, pVmxTransient));
5954	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, vmxHCExitHlt(pVCpu, pVmxTransient));
5955	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, vmxHCExitInvd(pVCpu, pVmxTransient));
5956	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, vmxHCExitInvlpg(pVCpu, pVmxTransient));
5957	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, vmxHCExitMtf(pVCpu, pVmxTransient));
5958	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, vmxHCExitPause(pVCpu, pVmxTransient));
5959	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, vmxHCExitWbinvd(pVCpu, pVmxTransient));
5960	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, vmxHCExitXsetbv(pVCpu, pVmxTransient));
5961	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, vmxHCExitInvpcid(pVCpu, pVmxTransient));
5962	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, vmxHCExitGetsec(pVCpu, pVmxTransient));
5963	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, vmxHCExitRdpmc(pVCpu, pVmxTransient));
5964	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5965	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, vmxHCExitVmclear(pVCpu, pVmxTransient));
5966	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, vmxHCExitVmlaunch(pVCpu, pVmxTransient));
5967	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, vmxHCExitVmptrld(pVCpu, pVmxTransient));
5968	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, vmxHCExitVmptrst(pVCpu, pVmxTransient));
5969	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, vmxHCExitVmread(pVCpu, pVmxTransient));
5970	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, vmxHCExitVmwrite(pVCpu, pVmxTransient));
5971	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, vmxHCExitVmresume(pVCpu, pVmxTransient));
5972	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, vmxHCExitVmxoff(pVCpu, pVmxTransient));
5973	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, vmxHCExitVmxon(pVCpu, pVmxTransient));
5974	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, vmxHCExitInvvpid(pVCpu, pVmxTransient));
5975	#else
5976	case VMX_EXIT_VMCLEAR:
5977	case VMX_EXIT_VMLAUNCH:
5978	case VMX_EXIT_VMPTRLD:
5979	case VMX_EXIT_VMPTRST:
5980	case VMX_EXIT_VMREAD:
5981	case VMX_EXIT_VMRESUME:
5982	case VMX_EXIT_VMWRITE:
5983	case VMX_EXIT_VMXOFF:
5984	case VMX_EXIT_VMXON:
5985	case VMX_EXIT_INVVPID:
5986	return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
5987	#endif
5988	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5989	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, vmxHCExitInvept(pVCpu, pVmxTransient));
5990	#else
5991	case VMX_EXIT_INVEPT: return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
5992	#endif
5993
5994	case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFault(pVCpu, pVmxTransient);
5995	case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindow(pVCpu, pVmxTransient);
5996	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
5997
5998	case VMX_EXIT_INIT_SIGNAL:
5999	case VMX_EXIT_SIPI:
6000	case VMX_EXIT_IO_SMI:
6001	case VMX_EXIT_SMI:
6002	case VMX_EXIT_ERR_MSR_LOAD:
6003	case VMX_EXIT_ERR_MACHINE_CHECK:
6004	case VMX_EXIT_PML_FULL:
6005	case VMX_EXIT_VIRTUALIZED_EOI:
6006	case VMX_EXIT_GDTR_IDTR_ACCESS:
6007	case VMX_EXIT_LDTR_TR_ACCESS:
6008	case VMX_EXIT_APIC_WRITE:
6009	case VMX_EXIT_RDRAND:
6010	case VMX_EXIT_RSM:
6011	case VMX_EXIT_VMFUNC:
6012	case VMX_EXIT_ENCLS:
6013	case VMX_EXIT_RDSEED:
6014	case VMX_EXIT_XSAVES:
6015	case VMX_EXIT_XRSTORS:
6016	case VMX_EXIT_UMWAIT:
6017	case VMX_EXIT_TPAUSE:
6018	case VMX_EXIT_LOADIWKEY:
6019	default:
6020	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
6021	}
6022	#undef VMEXIT_CALL_RET
6023	}
6024	#endif /* !HMVMX_USE_FUNCTION_TABLE */
6025
6026
6027	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6028	/**
6029	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
6030	*
6031	* @returns Strict VBox status code (i.e. informational status codes too).
6032	* @param pVCpu The cross context virtual CPU structure.
6033	* @param pVmxTransient The VMX-transient structure.
6034	*/
6035	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6036	{
6037	#ifdef DEBUG_ramshankar
6038	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
6039	do { \
6040	if (a_fSave != 0) \
6041	vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); \
6042	VBOXSTRICTRC rcStrict = a_CallExpr; \
6043	return rcStrict; \
6044	} while (0)
6045	#else
6046	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
6047	#endif
6048
6049	uint32_t const uExitReason = pVmxTransient->uExitReason;
6050	switch (uExitReason)
6051	{
6052	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6053	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, vmxHCExitEptMisconfigNested(pVCpu, pVmxTransient));
6054	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, vmxHCExitEptViolationNested(pVCpu, pVmxTransient));
6055	# else
6056	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, vmxHCExitEptMisconfig(pVCpu, pVmxTransient));
6057	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, vmxHCExitEptViolation(pVCpu, pVmxTransient));
6058	# endif
6059	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, vmxHCExitXcptOrNmiNested(pVCpu, pVmxTransient));
6060	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, vmxHCExitIoInstrNested(pVCpu, pVmxTransient));
6061	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, vmxHCExitHltNested(pVCpu, pVmxTransient));
6062
6063	/*
6064	* We shouldn't direct host physical interrupts to the nested-guest.
6065	*/
6066	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, vmxHCExitExtInt(pVCpu, pVmxTransient));
6067
6068	/*
6069	* Instructions that cause VM-exits unconditionally or the condition is
6070	* always taken solely from the nested hypervisor (meaning if the VM-exit
6071	* happens, it's guaranteed to be a nested-guest VM-exit).
6072	*
6073	* - Provides VM-exit instruction length ONLY.
6074	*/
6075	case VMX_EXIT_CPUID: /* Unconditional. */
6076	case VMX_EXIT_VMCALL:
6077	case VMX_EXIT_GETSEC:
6078	case VMX_EXIT_INVD:
6079	case VMX_EXIT_XSETBV:
6080	case VMX_EXIT_VMLAUNCH:
6081	case VMX_EXIT_VMRESUME:
6082	case VMX_EXIT_VMXOFF:
6083	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
6084	case VMX_EXIT_VMFUNC:
6085	VMEXIT_CALL_RET(0, vmxHCExitInstrNested(pVCpu, pVmxTransient));
6086
6087	/*
6088	* Instructions that cause VM-exits unconditionally or the condition is
6089	* always taken solely from the nested hypervisor (meaning if the VM-exit
6090	* happens, it's guaranteed to be a nested-guest VM-exit).
6091	*
6092	* - Provides VM-exit instruction length.
6093	* - Provides VM-exit information.
6094	* - Optionally provides Exit qualification.
6095	*
6096	* Since Exit qualification is 0 for all VM-exits where it is not
6097	* applicable, reading and passing it to the guest should produce
6098	* defined behavior.
6099	*
6100	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
6101	*/
6102	case VMX_EXIT_INVEPT: /* Unconditional. */
6103	case VMX_EXIT_INVVPID:
6104	case VMX_EXIT_VMCLEAR:
6105	case VMX_EXIT_VMPTRLD:
6106	case VMX_EXIT_VMPTRST:
6107	case VMX_EXIT_VMXON:
6108	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
6109	case VMX_EXIT_LDTR_TR_ACCESS:
6110	case VMX_EXIT_RDRAND:
6111	case VMX_EXIT_RDSEED:
6112	case VMX_EXIT_XSAVES:
6113	case VMX_EXIT_XRSTORS:
6114	case VMX_EXIT_UMWAIT:
6115	case VMX_EXIT_TPAUSE:
6116	VMEXIT_CALL_RET(0, vmxHCExitInstrWithInfoNested(pVCpu, pVmxTransient));
6117
6118	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, vmxHCExitRdtscNested(pVCpu, pVmxTransient));
6119	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, vmxHCExitRdtscpNested(pVCpu, pVmxTransient));
6120	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, vmxHCExitRdmsrNested(pVCpu, pVmxTransient));
6121	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, vmxHCExitWrmsrNested(pVCpu, pVmxTransient));
6122	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, vmxHCExitInvlpgNested(pVCpu, pVmxTransient));
6123	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, vmxHCExitInvpcidNested(pVCpu, pVmxTransient));
6124	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, vmxHCExitTaskSwitchNested(pVCpu, pVmxTransient));
6125	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, vmxHCExitWbinvdNested(pVCpu, pVmxTransient));
6126	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, vmxHCExitMtfNested(pVCpu, pVmxTransient));
6127	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, vmxHCExitApicAccessNested(pVCpu, pVmxTransient));
6128	case VMX_EXIT_APIC_WRITE: VMEXIT_CALL_RET(0, vmxHCExitApicWriteNested(pVCpu, pVmxTransient));
6129	case VMX_EXIT_VIRTUALIZED_EOI: VMEXIT_CALL_RET(0, vmxHCExitVirtEoiNested(pVCpu, pVmxTransient));
6130	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, vmxHCExitMovCRxNested(pVCpu, pVmxTransient));
6131	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, vmxHCExitIntWindowNested(pVCpu, pVmxTransient));
6132	case VMX_EXIT_NMI_WINDOW: VMEXIT_CALL_RET(0, vmxHCExitNmiWindowNested(pVCpu, pVmxTransient));
6133	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, vmxHCExitTprBelowThresholdNested(pVCpu, pVmxTransient));
6134	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, vmxHCExitMwaitNested(pVCpu, pVmxTransient));
6135	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, vmxHCExitMonitorNested(pVCpu, pVmxTransient));
6136	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, vmxHCExitPauseNested(pVCpu, pVmxTransient));
6137
6138	case VMX_EXIT_PREEMPT_TIMER:
6139	{
6140	/** @todo NSTVMX: Preempt timer. */
6141	VMEXIT_CALL_RET(0, vmxHCExitPreemptTimer(pVCpu, pVmxTransient));
6142	}
6143
6144	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, vmxHCExitMovDRxNested(pVCpu, pVmxTransient));
6145	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, vmxHCExitRdpmcNested(pVCpu, pVmxTransient));
6146
6147	case VMX_EXIT_VMREAD:
6148	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, vmxHCExitVmreadVmwriteNested(pVCpu, pVmxTransient));
6149
6150	case VMX_EXIT_TRIPLE_FAULT: VMEXIT_CALL_RET(0, vmxHCExitTripleFaultNested(pVCpu, pVmxTransient));
6151	case VMX_EXIT_ERR_INVALID_GUEST_STATE: VMEXIT_CALL_RET(0, vmxHCExitErrInvalidGuestStateNested(pVCpu, pVmxTransient));
6152
6153	case VMX_EXIT_INIT_SIGNAL:
6154	case VMX_EXIT_SIPI:
6155	case VMX_EXIT_IO_SMI:
6156	case VMX_EXIT_SMI:
6157	case VMX_EXIT_ERR_MSR_LOAD:
6158	case VMX_EXIT_ERR_MACHINE_CHECK:
6159	case VMX_EXIT_PML_FULL:
6160	case VMX_EXIT_RSM:
6161	default:
6162	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
6163	}
6164	#undef VMEXIT_CALL_RET
6165	}
6166	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6167
6168
6169	/** @name VM-exit helpers.
6170	* @{
6171	*/
6172	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6173	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6174	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6175
6176	/** Macro for VM-exits called unexpectedly. */
6177	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
6178	do { \
6179	VCPU_2_VMXSTATE((a_pVCpu)).u32HMError = (a_HmError); \
6180	return VERR_VMX_UNEXPECTED_EXIT; \
6181	} while (0)
6182
6183	#ifdef VBOX_STRICT
6184	# ifndef IN_NEM_DARWIN
6185	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6186	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
6187	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6188
6189	# define HMVMX_ASSERT_PREEMPT_CPUID() \
6190	do { \
6191	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6192	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6193	} while (0)
6194
6195	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
6196	do { \
6197	AssertPtr((a_pVCpu)); \
6198	AssertPtr((a_pVmxTransient)); \
6199	Assert( (a_pVmxTransient)->fVMEntryFailed == false \
6200	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \
6201	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \
6202	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \
6203	Assert((a_pVmxTransient)->pVmcsInfo); \
6204	Assert(ASMIntAreEnabled()); \
6205	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
6206	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
6207	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
6208	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
6209	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6210	HMVMX_ASSERT_PREEMPT_CPUID(); \
6211	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6212	} while (0)
6213	# else
6214	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() do { } while(0)
6215	# define HMVMX_ASSERT_PREEMPT_CPUID() do { } while(0)
6216	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
6217	do { \
6218	AssertPtr((a_pVCpu)); \
6219	AssertPtr((a_pVmxTransient)); \
6220	Assert( (a_pVmxTransient)->fVMEntryFailed == false \
6221	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \
6222	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \
6223	\|\| (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \
6224	Assert((a_pVmxTransient)->pVmcsInfo); \
6225	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
6226	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6227	} while (0)
6228	# endif
6229
6230	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
6231	do { \
6232	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
6233	Assert((a_pVmxTransient)->fIsNestedGuest); \
6234	} while (0)
6235
6236	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
6237	do { \
6238	Log4Func(("\n")); \
6239	} while (0)
6240	#else
6241	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
6242	do { \
6243	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6244	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
6245	} while (0)
6246
6247	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
6248	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
6249
6250	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
6251	#endif
6252
6253	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6254	/** Macro that does the necessary privilege checks and intercepted VM-exits for
6255	* guests that attempted to execute a VMX instruction. */
6256	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
6257	do \
6258	{ \
6259	VBOXSTRICTRC rcStrictTmp = vmxHCCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
6260	if (rcStrictTmp == VINF_SUCCESS) \
6261	{ /* likely */ } \
6262	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
6263	{ \
6264	Assert((a_pVCpu)->hm.s.Event.fPending); \
6265	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
6266	return VINF_SUCCESS; \
6267	} \
6268	else \
6269	{ \
6270	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
6271	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
6272	} \
6273	} while (0)
6274
6275	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
6276	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
6277	do \
6278	{ \
6279	VBOXSTRICTRC rcStrictTmp = vmxHCDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
6280	(a_pGCPtrEffAddr)); \
6281	if (rcStrictTmp == VINF_SUCCESS) \
6282	{ /* likely */ } \
6283	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
6284	{ \
6285	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
6286	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
6287	NOREF(uXcptTmp); \
6288	return VINF_SUCCESS; \
6289	} \
6290	else \
6291	{ \
6292	Log4Func(("vmxHCDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
6293	return rcStrictTmp; \
6294	} \
6295	} while (0)
6296	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6297
6298
6299	/**
6300	* Advances the guest RIP by the specified number of bytes.
6301	*
6302	* @param pVCpu The cross context virtual CPU structure.
6303	* @param cbInstr Number of bytes to advance the RIP by.
6304	*
6305	* @remarks No-long-jump zone!!!
6306	*/
6307	DECLINLINE(void) vmxHCAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
6308	{
6309	CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI);
6310
6311	/*
6312	* Advance RIP.
6313	*
6314	* The upper 32 bits are only set when in 64-bit mode, so we have to detect
6315	* when the addition causes a "carry" into the upper half and check whether
6316	* we're in 64-bit and can go on with it or wether we should zap the top
6317	* half. (Note! The 8086, 80186 and 80286 emulation is done exclusively in
6318	* IEM, so we don't need to bother with pre-386 16-bit wraparound.)
6319	*
6320	* See PC wrap around tests in bs3-cpu-weird-1.
6321	*/
6322	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
6323	uint64_t const uRipNext = uRipPrev + cbInstr;
6324	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & RT_BIT_64(32))
6325	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6326	pVCpu->cpum.GstCtx.rip = uRipNext;
6327	else
6328	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
6329
6330	/*
6331	* Clear RF and interrupt shadowing.
6332	*/
6333	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_RF \| X86_EFL_TF))))
6334	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_INHIBIT_SHADOW;
6335	else
6336	{
6337	if ((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_RF \| X86_EFL_TF)) == X86_EFL_TF)
6338	{
6339	/** @todo \#DB - single step. */
6340	}
6341	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
6342	}
6343	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
6344
6345	/* Mark both RIP and RFLAGS as updated. */
6346	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
6347	}
6348
6349
6350	/**
6351	* Advances the guest RIP after reading it from the VMCS.
6352	*
6353	* @returns VBox status code, no informational status codes.
6354	* @param pVCpu The cross context virtual CPU structure.
6355	* @param pVmxTransient The VMX-transient structure.
6356	*
6357	* @remarks No-long-jump zone!!!
6358	*/
6359	static int vmxHCAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6360	{
6361	vmxHCReadToTransientSlow<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
6362	/** @todo consider template here after checking callers. */
6363	int rc = vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
6364	AssertRCReturn(rc, rc);
6365
6366	vmxHCAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
6367	return VINF_SUCCESS;
6368	}
6369
6370
6371	/**
6372	* Handle a condition that occurred while delivering an event through the guest or
6373	* nested-guest IDT.
6374	*
6375	* @returns Strict VBox status code (i.e. informational status codes too).
6376	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6377	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6378	* to continue execution of the guest which will delivery the \#DF.
6379	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6380	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6381	*
6382	* @param pVCpu The cross context virtual CPU structure.
6383	* @param pVmxTransient The VMX-transient structure.
6384	*
6385	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6386	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
6387	* is due to an EPT violation, PML full or SPP-related event.
6388	*
6389	* @remarks No-long-jump zone!!!
6390	*/
6391	static VBOXSTRICTRC vmxHCCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6392	{
6393	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
6394	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
6395	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
6396	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
6397	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
6398	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
6399
6400	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6401	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6402	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
6403	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
6404	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
6405	{
6406	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
6407	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
6408
6409	/*
6410	* If the event was a software interrupt (generated with INT n) or a software exception
6411	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6412	* can handle the VM-exit and continue guest execution which will re-execute the
6413	* instruction rather than re-injecting the exception, as that can cause premature
6414	* trips to ring-3 before injection and involve TRPM which currently has no way of
6415	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6416	* the problem).
6417	*/
6418	IEMXCPTRAISE enmRaise;
6419	IEMXCPTRAISEINFO fRaiseInfo;
6420	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6421	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6422	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6423	{
6424	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6425	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6426	}
6427	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
6428	{
6429	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
6430	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
6431	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
6432
6433	uint32_t const fIdtVectorFlags = vmxHCGetIemXcptFlags(uIdtVector, uIdtVectorType);
6434	uint32_t const fExitVectorFlags = vmxHCGetIemXcptFlags(uExitVector, uExitVectorType);
6435
6436	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6437
6438	/* Determine a vectoring #PF condition, see comment in vmxHCExitXcptPF(). */
6439	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6440	{
6441	pVmxTransient->fVectoringPF = true;
6442	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6443	}
6444	}
6445	else
6446	{
6447	/*
6448	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6449	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6450	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
6451	*/
6452	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6453	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6454	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6455	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6456	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6457	}
6458
6459	/*
6460	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6461	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6462	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6463	* subsequent VM-entry would fail, see @bugref{7445}.
6464	*
6465	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
6466	*/
6467	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6468	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
6469	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
6470	&& CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
6471	CPUMClearInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
6472
6473	switch (enmRaise)
6474	{
6475	case IEMXCPTRAISE_CURRENT_XCPT:
6476	{
6477	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
6478	Assert(rcStrict == VINF_SUCCESS);
6479	break;
6480	}
6481
6482	case IEMXCPTRAISE_PREV_EVENT:
6483	{
6484	uint32_t u32ErrCode;
6485	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
6486	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6487	else
6488	u32ErrCode = 0;
6489
6490	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see vmxHCExitXcptPF(). */
6491	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflect);
6492	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */, u32ErrCode,
6493	pVCpu->cpum.GstCtx.cr2);
6494
6495	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
6496	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode));
6497	Assert(rcStrict == VINF_SUCCESS);
6498	break;
6499	}
6500
6501	case IEMXCPTRAISE_REEXEC_INSTR:
6502	Assert(rcStrict == VINF_SUCCESS);
6503	break;
6504
6505	case IEMXCPTRAISE_DOUBLE_FAULT:
6506	{
6507	/*
6508	* Determine a vectoring double #PF condition. Used later, when PGM evaluates the
6509	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6510	*/
6511	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6512	{
6513	pVmxTransient->fVectoringDoublePF = true;
6514	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
6515	pVCpu->cpum.GstCtx.cr2));
6516	rcStrict = VINF_SUCCESS;
6517	}
6518	else
6519	{
6520	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectConvertDF);
6521	vmxHCSetPendingXcptDF(pVCpu);
6522	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
6523	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
6524	rcStrict = VINF_HM_DOUBLE_FAULT;
6525	}
6526	break;
6527	}
6528
6529	case IEMXCPTRAISE_TRIPLE_FAULT:
6530	{
6531	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
6532	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
6533	rcStrict = VINF_EM_RESET;
6534	break;
6535	}
6536
6537	case IEMXCPTRAISE_CPU_HANG:
6538	{
6539	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6540	rcStrict = VERR_EM_GUEST_CPU_HANG;
6541	break;
6542	}
6543
6544	default:
6545	{
6546	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6547	rcStrict = VERR_VMX_IPE_2;
6548	break;
6549	}
6550	}
6551	}
6552	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
6553	&& !CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
6554	{
6555	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
6556	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
6557	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
6558	{
6559	/*
6560	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
6561	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
6562	* that virtual NMIs remain blocked until the IRET execution is completed.
6563	*
6564	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
6565	*/
6566	CPUMSetInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
6567	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
6568	}
6569	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
6570	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
6571	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
6572	{
6573	/*
6574	* Execution of IRET caused an EPT violation, page-modification log-full event or
6575	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
6576	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
6577	* that virtual NMIs remain blocked until the IRET execution is completed.
6578	*
6579	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
6580	*/
6581	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
6582	{
6583	CPUMSetInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
6584	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
6585	}
6586	}
6587	}
6588
6589	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6590	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6591	return rcStrict;
6592	}
6593
6594
6595	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6596	/**
6597	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6598	* guest attempting to execute a VMX instruction.
6599	*
6600	* @returns Strict VBox status code (i.e. informational status codes too).
6601	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6602	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6603	*
6604	* @param pVCpu The cross context virtual CPU structure.
6605	* @param uExitReason The VM-exit reason.
6606	*
6607	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6608	* @remarks No-long-jump zone!!!
6609	*/
6610	static VBOXSTRICTRC vmxHCCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
6611	{
6612	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6613	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6614
6615	/*
6616	* The physical CPU would have already checked the CPU mode/code segment.
6617	* We shall just assert here for paranoia.
6618	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
6619	*/
6620	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6621	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6622	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
6623
6624	if (uExitReason == VMX_EXIT_VMXON)
6625	{
6626	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6627
6628	/*
6629	* We check CR4.VMXE because it is required to be always set while in VMX operation
6630	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
6631	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6632	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6633	*/
6634	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6635	{
6636	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6637	vmxHCSetPendingXcptUD(pVCpu);
6638	return VINF_HM_PENDING_XCPT;
6639	}
6640	}
6641	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6642	{
6643	/*
6644	* The guest has not entered VMX operation but attempted to execute a VMX instruction
6645	* (other than VMXON), we need to raise a #UD.
6646	*/
6647	Log4Func(("Not in VMX root mode -> #UD\n"));
6648	vmxHCSetPendingXcptUD(pVCpu);
6649	return VINF_HM_PENDING_XCPT;
6650	}
6651
6652	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
6653	return VINF_SUCCESS;
6654	}
6655
6656
6657	/**
6658	* Decodes the memory operand of an instruction that caused a VM-exit.
6659	*
6660	* The Exit qualification field provides the displacement field for memory
6661	* operand instructions, if any.
6662	*
6663	* @returns Strict VBox status code (i.e. informational status codes too).
6664	* @retval VINF_SUCCESS if the operand was successfully decoded.
6665	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6666	* operand.
6667	* @param pVCpu The cross context virtual CPU structure.
6668	* @param uExitInstrInfo The VM-exit instruction information field.
6669	* @param enmMemAccess The memory operand's access type (read or write).
6670	* @param GCPtrDisp The instruction displacement field, if any. For
6671	* RIP-relative addressing pass RIP + displacement here.
6672	* @param pGCPtrMem Where to store the effective destination memory address.
6673	*
6674	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
6675	* virtual-8086 mode hence skips those checks while verifying if the
6676	* segment is valid.
6677	*/
6678	static VBOXSTRICTRC vmxHCDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6679	PRTGCPTR pGCPtrMem)
6680	{
6681	Assert(pGCPtrMem);
6682	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6683	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6684	\| CPUMCTX_EXTRN_CR0);
6685
6686	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6687	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6688	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6689
6690	VMXEXITINSTRINFO ExitInstrInfo;
6691	ExitInstrInfo.u = uExitInstrInfo;
6692	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6693	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6694	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6695	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6696	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6697	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6698	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6699	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6700	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6701
6702	/*
6703	* Validate instruction information.
6704	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6705	*/
6706	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6707	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6708	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6709	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6710	AssertLogRelMsgReturn(fIsMemOperand,
6711	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6712
6713	/*
6714	* Compute the complete effective address.
6715	*
6716	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6717	* See AMD spec. 4.5.2 "Segment Registers".
6718	*/
6719	RTGCPTR GCPtrMem = GCPtrDisp;
6720	if (fBaseRegValid)
6721	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6722	if (fIdxRegValid)
6723	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6724
6725	RTGCPTR const GCPtrOff = GCPtrMem;
6726	if ( !fIsLongMode
6727	\|\| iSegReg >= X86_SREG_FS)
6728	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6729	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6730
6731	/*
6732	* Validate effective address.
6733	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6734	*/
6735	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6736	Assert(cbAccess > 0);
6737	if (fIsLongMode)
6738	{
6739	if (X86_IS_CANONICAL(GCPtrMem))
6740	{
6741	*pGCPtrMem = GCPtrMem;
6742	return VINF_SUCCESS;
6743	}
6744
6745	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6746	* "Data Limit Checks in 64-bit Mode". */
6747	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6748	vmxHCSetPendingXcptGP(pVCpu, 0);
6749	return VINF_HM_PENDING_XCPT;
6750	}
6751
6752	/*
6753	* This is a watered down version of iemMemApplySegment().
6754	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6755	* and segment CPL/DPL checks are skipped.
6756	*/
6757	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6758	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6759	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6760
6761	/* Check if the segment is present and usable. */
6762	if ( pSel->Attr.n.u1Present
6763	&& !pSel->Attr.n.u1Unusable)
6764	{
6765	Assert(pSel->Attr.n.u1DescType);
6766	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6767	{
6768	/* Check permissions for the data segment. */
6769	if ( enmMemAccess == VMXMEMACCESS_WRITE
6770	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6771	{
6772	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6773	vmxHCSetPendingXcptGP(pVCpu, iSegReg);
6774	return VINF_HM_PENDING_XCPT;
6775	}
6776
6777	/* Check limits if it's a normal data segment. */
6778	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6779	{
6780	if ( GCPtrFirst32 > pSel->u32Limit
6781	\|\| GCPtrLast32 > pSel->u32Limit)
6782	{
6783	Log4Func(("Data segment limit exceeded. "
6784	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6785	GCPtrLast32, pSel->u32Limit));
6786	if (iSegReg == X86_SREG_SS)
6787	vmxHCSetPendingXcptSS(pVCpu, 0);
6788	else
6789	vmxHCSetPendingXcptGP(pVCpu, 0);
6790	return VINF_HM_PENDING_XCPT;
6791	}
6792	}
6793	else
6794	{
6795	/* Check limits if it's an expand-down data segment.
6796	Note! The upper boundary is defined by the B bit, not the G bit! */
6797	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6798	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6799	{
6800	Log4Func(("Expand-down data segment limit exceeded. "
6801	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6802	GCPtrLast32, pSel->u32Limit));
6803	if (iSegReg == X86_SREG_SS)
6804	vmxHCSetPendingXcptSS(pVCpu, 0);
6805	else
6806	vmxHCSetPendingXcptGP(pVCpu, 0);
6807	return VINF_HM_PENDING_XCPT;
6808	}
6809	}
6810	}
6811	else
6812	{
6813	/* Check permissions for the code segment. */
6814	if ( enmMemAccess == VMXMEMACCESS_WRITE
6815	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6816	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6817	{
6818	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6819	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6820	vmxHCSetPendingXcptGP(pVCpu, 0);
6821	return VINF_HM_PENDING_XCPT;
6822	}
6823
6824	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6825	if ( GCPtrFirst32 > pSel->u32Limit
6826	\|\| GCPtrLast32 > pSel->u32Limit)
6827	{
6828	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6829	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6830	if (iSegReg == X86_SREG_SS)
6831	vmxHCSetPendingXcptSS(pVCpu, 0);
6832	else
6833	vmxHCSetPendingXcptGP(pVCpu, 0);
6834	return VINF_HM_PENDING_XCPT;
6835	}
6836	}
6837	}
6838	else
6839	{
6840	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6841	vmxHCSetPendingXcptGP(pVCpu, 0);
6842	return VINF_HM_PENDING_XCPT;
6843	}
6844
6845	*pGCPtrMem = GCPtrMem;
6846	return VINF_SUCCESS;
6847	}
6848	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6849
6850
6851	/**
6852	* VM-exit helper for LMSW.
6853	*/
6854	static VBOXSTRICTRC vmxHCExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
6855	{
6856	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
6857	AssertRCReturn(rc, rc);
6858
6859	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
6860	AssertMsg( rcStrict == VINF_SUCCESS
6861	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6862
6863	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
6864	if (rcStrict == VINF_IEM_RAISED_XCPT)
6865	{
6866	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6867	rcStrict = VINF_SUCCESS;
6868	}
6869
6870	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitLmsw);
6871	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6872	return rcStrict;
6873	}
6874
6875
6876	/**
6877	* VM-exit helper for CLTS.
6878	*/
6879	static VBOXSTRICTRC vmxHCExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
6880	{
6881	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
6882	AssertRCReturn(rc, rc);
6883
6884	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
6885	AssertMsg( rcStrict == VINF_SUCCESS
6886	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6887
6888	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
6889	if (rcStrict == VINF_IEM_RAISED_XCPT)
6890	{
6891	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6892	rcStrict = VINF_SUCCESS;
6893	}
6894
6895	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitClts);
6896	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6897	return rcStrict;
6898	}
6899
6900
6901	/**
6902	* VM-exit helper for MOV from CRx (CRx read).
6903	*/
6904	static VBOXSTRICTRC vmxHCExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
6905	{
6906	Assert(iCrReg < 16);
6907	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
6908
6909	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
6910	AssertRCReturn(rc, rc);
6911
6912	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6913	AssertMsg( rcStrict == VINF_SUCCESS
6914	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6915
6916	if (iGReg == X86_GREG_xSP)
6917	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
6918	else
6919	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
6920	#ifdef VBOX_WITH_STATISTICS
6921	switch (iCrReg)
6922	{
6923	case 0: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Read); break;
6924	case 2: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Read); break;
6925	case 3: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Read); break;
6926	case 4: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Read); break;
6927	case 8: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Read); break;
6928	}
6929	#endif
6930	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
6931	return rcStrict;
6932	}
6933
6934
6935	/**
6936	* VM-exit helper for MOV to CRx (CRx write).
6937	*/
6938	static VBOXSTRICTRC vmxHCExitMovToCrX(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
6939	{
6940	HMVMX_CPUMCTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6941
6942	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6943	AssertMsg( rcStrict == VINF_SUCCESS
6944	\|\| rcStrict == VINF_IEM_RAISED_XCPT
6945	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6946
6947	switch (iCrReg)
6948	{
6949	case 0:
6950	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
6951	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
6952	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Write);
6953	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
6954	break;
6955
6956	case 2:
6957	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Write);
6958	/* Nothing to do here, CR2 it's not part of the VMCS. */
6959	break;
6960
6961	case 3:
6962	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
6963	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Write);
6964	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
6965	break;
6966
6967	case 4:
6968	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
6969	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Write);
6970	#ifndef IN_NEM_DARWIN
6971	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
6972	pVCpu->cpum.GstCtx.cr4, pVCpu->hmr0.s.fLoadSaveGuestXcr0));
6973	#else
6974	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr4));
6975	#endif
6976	break;
6977
6978	case 8:
6979	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
6980	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
6981	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Write);
6982	break;
6983
6984	default:
6985	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
6986	break;
6987	}
6988
6989	if (rcStrict == VINF_IEM_RAISED_XCPT)
6990	{
6991	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6992	rcStrict = VINF_SUCCESS;
6993	}
6994	return rcStrict;
6995	}
6996
6997
6998	/**
6999	* VM-exit exception handler for \#PF (Page-fault exception).
7000	*
7001	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
7002	*/
7003	static VBOXSTRICTRC vmxHCExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7004	{
7005	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7006	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
7007
7008	#ifndef IN_NEM_DARWIN
7009	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7010	if (!VM_IS_VMX_NESTED_PAGING(pVM))
7011	{ /* likely */ }
7012	else
7013	#endif
7014	{
7015	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF) && !defined(IN_NEM_DARWIN)
7016	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hmr0.s.fUsingDebugLoop);
7017	#endif
7018	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7019	if (!pVmxTransient->fVectoringDoublePF)
7020	{
7021	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
7022	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
7023	}
7024	else
7025	{
7026	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7027	Assert(!pVmxTransient->fIsNestedGuest);
7028	vmxHCSetPendingXcptDF(pVCpu);
7029	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
7030	}
7031	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
7032	return VINF_SUCCESS;
7033	}
7034
7035	Assert(!pVmxTransient->fIsNestedGuest);
7036
7037	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
7038	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
7039	if (pVmxTransient->fVectoringPF)
7040	{
7041	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
7042	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7043	}
7044
7045	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7046	AssertRCReturn(rc, rc);
7047
7048	Log4Func(("#PF: cs:rip=%#04x:%08RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pVCpu->cpum.GstCtx.cs.Sel,
7049	pVCpu->cpum.GstCtx.rip, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pVCpu->cpum.GstCtx.cr3));
7050
7051	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
7052	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, &pVCpu->cpum.GstCtx, (RTGCPTR)pVmxTransient->uExitQual);
7053
7054	Log4Func(("#PF: rc=%Rrc\n", rc));
7055	if (rc == VINF_SUCCESS)
7056	{
7057	/*
7058	* This is typically a shadow page table sync or a MMIO instruction. But we may have
7059	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
7060	*/
7061	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
7062	TRPMResetTrap(pVCpu);
7063	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPF);
7064	return rc;
7065	}
7066
7067	if (rc == VINF_EM_RAW_GUEST_TRAP)
7068	{
7069	if (!pVmxTransient->fVectoringDoublePF)
7070	{
7071	/* It's a guest page fault and needs to be reflected to the guest. */
7072	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
7073	TRPMResetTrap(pVCpu);
7074	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory #PF. */
7075	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
7076	uGstErrorCode, pVmxTransient->uExitQual);
7077	}
7078	else
7079	{
7080	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7081	TRPMResetTrap(pVCpu);
7082	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
7083	vmxHCSetPendingXcptDF(pVCpu);
7084	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
7085	}
7086
7087	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
7088	return VINF_SUCCESS;
7089	}
7090
7091	TRPMResetTrap(pVCpu);
7092	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPFEM);
7093	return rc;
7094	}
7095
7096
7097	/**
7098	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
7099	*
7100	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
7101	*/
7102	static VBOXSTRICTRC vmxHCExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7103	{
7104	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7105	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF);
7106
7107	int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CR0>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7108	AssertRCReturn(rc, rc);
7109
7110	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
7111	{
7112	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
7113	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
7114
7115	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
7116	* provides VM-exit instruction length. If this causes problem later,
7117	* disassemble the instruction like it's done on AMD-V. */
7118	int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7119	AssertRCReturn(rc2, rc2);
7120	return rc;
7121	}
7122
7123	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
7124	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
7125	return VINF_SUCCESS;
7126	}
7127
7128
7129	/**
7130	* VM-exit exception handler for \#BP (Breakpoint exception).
7131	*
7132	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
7133	*/
7134	static VBOXSTRICTRC vmxHCExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7135	{
7136	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7137	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP);
7138
7139	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7140	AssertRCReturn(rc, rc);
7141
7142	VBOXSTRICTRC rcStrict;
7143	if (!pVmxTransient->fIsNestedGuest)
7144	rcStrict = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx);
7145	else
7146	rcStrict = VINF_EM_RAW_GUEST_TRAP;
7147
7148	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
7149	{
7150	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
7151	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
7152	rcStrict = VINF_SUCCESS;
7153	}
7154
7155	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_DBG_BREAKPOINT);
7156	return rcStrict;
7157	}
7158
7159
7160	/**
7161	* VM-exit helper for split-lock access triggered \#AC exceptions.
7162	*/
7163	static VBOXSTRICTRC vmxHCHandleSplitLockAcXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7164	{
7165	/*
7166	* Check for debug/trace events and import state accordingly.
7167	*/
7168	if (!pVmxTransient->fIsNestedGuest)
7169	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestACSplitLock);
7170	else
7171	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatNestedExitACSplitLock);
7172	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7173	if ( !DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK)
7174	#ifndef IN_NEM_DARWIN
7175	&& !VBOXVMM_VMX_SPLIT_LOCK_ENABLED()
7176	#endif
7177	)
7178	{
7179	if (pVM->cCpus == 1)
7180	{
7181	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
7182	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK,
7183	HMVMX_CPUMCTX_XPCT_AC>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7184	#else
7185	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
7186	HMVMX_CPUMCTX_XPCT_AC>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7187	#endif
7188	AssertRCReturn(rc, rc);
7189	}
7190	}
7191	else
7192	{
7193	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
7194	HMVMX_CPUMCTX_XPCT_AC>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7195	AssertRCReturn(rc, rc);
7196
7197	VBOXVMM_XCPT_DF(pVCpu, &pVCpu->cpum.GstCtx);
7198
7199	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK))
7200	{
7201	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, DBGFEVENT_VMX_SPLIT_LOCK, DBGFEVENTCTX_HM, 0);
7202	if (rcStrict != VINF_SUCCESS)
7203	return rcStrict;
7204	}
7205	}
7206
7207	/*
7208	* Emulate the instruction.
7209	*
7210	* We have to ignore the LOCK prefix here as we must not retrigger the
7211	* detection on the host. This isn't all that satisfactory, though...
7212	*/
7213	if (pVM->cCpus == 1)
7214	{
7215	Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC\n", pVCpu->cpum.GstCtx.cs.Sel,
7216	pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
7217
7218	/** @todo For SMP configs we should do a rendezvous here. */
7219	VBOXSTRICTRC rcStrict = IEMExecOneIgnoreLock(pVCpu);
7220	if (rcStrict == VINF_SUCCESS)
7221	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
7222	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
7223	HM_CHANGED_GUEST_RIP
7224	\| HM_CHANGED_GUEST_RFLAGS
7225	\| HM_CHANGED_GUEST_GPRS_MASK
7226	\| HM_CHANGED_GUEST_CS
7227	\| HM_CHANGED_GUEST_SS);
7228	#else
7229	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
7230	#endif
7231	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7232	{
7233	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7234	rcStrict = VINF_SUCCESS;
7235	}
7236	return rcStrict;
7237	}
7238	Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC -> VINF_EM_EMULATE_SPLIT_LOCK\n",
7239	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
7240	return VINF_EM_EMULATE_SPLIT_LOCK;
7241	}
7242
7243
7244	/**
7245	* VM-exit exception handler for \#AC (Alignment-check exception).
7246	*
7247	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
7248	*/
7249	static VBOXSTRICTRC vmxHCExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7250	{
7251	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7252
7253	/*
7254	* Detect #ACs caused by host having enabled split-lock detection.
7255	* Emulate such instructions.
7256	*/
7257	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_XPCT_AC>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7258	AssertRCReturn(rc, rc);
7259	/** @todo detect split lock in cpu feature? */
7260	/** @todo r=ramshankar: is cpu feature detection really necessary since we are able
7261	* to detect the split-lock \#AC condition without it? More so since the
7262	* feature isn't cleanly detectable, see @bugref{10318#c125}. */
7263	if (vmxHCIsSplitLockAcXcpt(pVCpu))
7264	return vmxHCHandleSplitLockAcXcpt(pVCpu, pVmxTransient);
7265
7266	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC);
7267	Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 cpl=%d -> #AC\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7268	pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0, CPUMGetGuestCPL(pVCpu) ));
7269
7270	/* Re-inject it. We'll detect any nesting before getting here. */
7271	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
7272	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
7273	return VINF_SUCCESS;
7274	}
7275
7276
7277	/**
7278	* VM-exit exception handler for \#DB (Debug exception).
7279	*
7280	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
7281	*/
7282	static VBOXSTRICTRC vmxHCExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7283	{
7284	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7285	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB);
7286
7287	/*
7288	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
7289	*/
7290	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
7291
7292	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
7293	uint64_t const uDR6 = X86_DR6_INIT_VAL
7294	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
7295	\| X86_DR6_BD \| X86_DR6_BS));
7296	Log6Func(("uDR6=%#RX64 uExitQual=%#RX64\n", uDR6, pVmxTransient->uExitQual));
7297
7298	int rc;
7299	if (!pVmxTransient->fIsNestedGuest)
7300	{
7301	rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx, uDR6, VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
7302
7303	/*
7304	* Prevents stepping twice over the same instruction when the guest is stepping using
7305	* EFLAGS.TF and the hypervisor debugger is stepping using MTF.
7306	* Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
7307	*/
7308	if ( rc == VINF_EM_DBG_STEPPED
7309	&& (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
7310	{
7311	Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
7312	rc = VINF_EM_RAW_GUEST_TRAP;
7313	}
7314	}
7315	else
7316	rc = VINF_EM_RAW_GUEST_TRAP;
7317	Log6Func(("rc=%Rrc\n", rc));
7318	if (rc == VINF_EM_RAW_GUEST_TRAP)
7319	{
7320	/*
7321	* The exception was for the guest. Update DR6, DR7.GD and
7322	* IA32_DEBUGCTL.LBR before forwarding it.
7323	* See Intel spec. 27.1 "Architectural State before a VM-Exit"
7324	* and @sdmv3{077,622,17.2.3,Debug Status Register (DR6)}.
7325	*/
7326	#ifndef IN_NEM_DARWIN
7327	VMMRZCallRing3Disable(pVCpu);
7328	HM_DISABLE_PREEMPT(pVCpu);
7329
7330	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
7331	pVCpu->cpum.GstCtx.dr[6] \|= uDR6;
7332	if (CPUMIsGuestDebugStateActive(pVCpu))
7333	ASMSetDR6(pVCpu->cpum.GstCtx.dr[6]);
7334
7335	HM_RESTORE_PREEMPT();
7336	VMMRZCallRing3Enable(pVCpu);
7337	#else
7338	/** @todo */
7339	#endif
7340
7341	rc = vmxHCImportGuestState<CPUMCTX_EXTRN_DR7>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7342	AssertRCReturn(rc, rc);
7343
7344	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7345	pVCpu->cpum.GstCtx.dr[7] &= ~(uint64_t)X86_DR7_GD;
7346
7347	/* Paranoia. */
7348	pVCpu->cpum.GstCtx.dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
7349	pVCpu->cpum.GstCtx.dr[7] \|= X86_DR7_RA1_MASK;
7350
7351	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, pVCpu->cpum.GstCtx.dr[7]);
7352	AssertRC(rc);
7353
7354	/*
7355	* Raise #DB in the guest.
7356	*
7357	* It is important to reflect exactly what the VM-exit gave us (preserving the
7358	* interruption-type) rather than use vmxHCSetPendingXcptDB() as the #DB could've
7359	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
7360	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
7361	*
7362	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
7363	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7364	*/
7365	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
7366	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
7367	return VINF_SUCCESS;
7368	}
7369
7370	/*
7371	* Not a guest trap, must be a hypervisor related debug event then.
7372	* Update DR6 in case someone is interested in it.
7373	*/
7374	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
7375	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
7376	CPUMSetHyperDR6(pVCpu, uDR6);
7377
7378	return rc;
7379	}
7380
7381
7382	/**
7383	* Hacks its way around the lovely mesa driver's backdoor accesses.
7384	*
7385	* @sa hmR0SvmHandleMesaDrvGp.
7386	*/
7387	static int vmxHCHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
7388	{
7389	LogFunc(("cs:rip=%#04x:%08RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
7390	RT_NOREF(pCtx);
7391
7392	/* For now we'll just skip the instruction. */
7393	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7394	}
7395
7396
7397	/**
7398	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
7399	* backdoor logging w/o checking what it is running inside.
7400	*
7401	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
7402	* backdoor port and magic numbers loaded in registers.
7403	*
7404	* @returns true if it is, false if it isn't.
7405	* @sa hmR0SvmIsMesaDrvGp.
7406	*/
7407	DECLINLINE(bool) vmxHCIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
7408	{
7409	/* 0xed: IN eAX,dx */
7410	uint8_t abInstr[1];
7411	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
7412	return false;
7413
7414	/* Check that it is #GP(0). */
7415	if (pVmxTransient->uExitIntErrorCode != 0)
7416	return false;
7417
7418	/* Check magic and port. */
7419	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
7420	/Log(("vmxHCIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
7421	if (pCtx->rax != UINT32_C(0x564d5868))
7422	return false;
7423	if (pCtx->dx != UINT32_C(0x5658))
7424	return false;
7425
7426	/* Flat ring-3 CS. */
7427	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
7428	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
7429	/Log(("vmxHCIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
7430	if (pCtx->cs.Attr.n.u2Dpl != 3)
7431	return false;
7432	if (pCtx->cs.u64Base != 0)
7433	return false;
7434
7435	/* Check opcode. */
7436	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
7437	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
7438	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
7439	/Log(("vmxHCIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
7440	if (RT_FAILURE(rc))
7441	return false;
7442	if (abInstr[0] != 0xed)
7443	return false;
7444
7445	return true;
7446	}
7447
7448
7449	/**
7450	* VM-exit exception handler for \#GP (General-protection exception).
7451	*
7452	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
7453	*/
7454	static VBOXSTRICTRC vmxHCExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7455	{
7456	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7457	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP);
7458
7459	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7460	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7461	#ifndef IN_NEM_DARWIN
7462	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
7463	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
7464	{ /* likely */ }
7465	else
7466	#endif
7467	{
7468	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7469	# ifndef IN_NEM_DARWIN
7470	Assert(pVCpu->hmr0.s.fUsingDebugLoop \|\| VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
7471	# else
7472	Assert(/pVCpu->hmr0.s.fUsingDebugLoop \|\|/ VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
7473	# endif
7474	#endif
7475	/*
7476	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
7477	* executing a nested-guest, reflect #GP to the guest or nested-guest.
7478	*/
7479	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
7480	AssertRCReturn(rc, rc);
7481	Log4Func(("Gst: cs:rip=%#04x:%08RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
7482	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
7483
7484	if ( pVmxTransient->fIsNestedGuest
7485	\|\| !VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv
7486	\|\| !vmxHCIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
7487	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
7488	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
7489	else
7490	rc = vmxHCHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
7491	return rc;
7492	}
7493
7494	#ifndef IN_NEM_DARWIN
7495	Assert(CPUMIsGuestInRealModeEx(pCtx));
7496	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest);
7497	Assert(!pVmxTransient->fIsNestedGuest);
7498
7499	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
7500	AssertRCReturn(rc, rc);
7501
7502	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7503	if (rcStrict == VINF_SUCCESS)
7504	{
7505	if (!CPUMIsGuestInRealModeEx(pCtx))
7506	{
7507	/*
7508	* The guest is no longer in real-mode, check if we can continue executing the
7509	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
7510	*/
7511	pVmcsInfoShared->RealMode.fRealOnV86Active = false;
7512	if (HMCanExecuteVmxGuest(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx))
7513	{
7514	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
7515	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
7516	}
7517	else
7518	{
7519	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
7520	rcStrict = VINF_EM_RESCHEDULE;
7521	}
7522	}
7523	else
7524	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
7525	}
7526	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7527	{
7528	rcStrict = VINF_SUCCESS;
7529	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7530	}
7531	return VBOXSTRICTRC_VAL(rcStrict);
7532	#endif
7533	}
7534
7535
7536	/**
7537	* VM-exit exception handler for \#DE (Divide Error).
7538	*
7539	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
7540	*/
7541	static VBOXSTRICTRC vmxHCExitXcptDE(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7542	{
7543	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7544	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE);
7545
7546	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7547	AssertRCReturn(rc, rc);
7548
7549	if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE)
7550	{
7551	rc = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx);
7552	Assert(rc == VINF_SUCCESS /* restart instr / \|\| rc == VERR_NOT_FOUND / deliver exception */);
7553	}
7554	else
7555	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
7556
7557	/* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
7558	if (RT_FAILURE(rc))
7559	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
7560	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
7561	return VINF_SUCCESS;
7562	}
7563
7564
7565	/**
7566	* VM-exit exception handler wrapper for all other exceptions that are not handled
7567	* by a specific handler.
7568	*
7569	* This simply re-injects the exception back into the VM without any special
7570	* processing.
7571	*
7572	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
7573	*/
7574	static VBOXSTRICTRC vmxHCExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7575	{
7576	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7577
7578	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7579	# ifndef IN_NEM_DARWIN
7580	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7581	AssertMsg(pVCpu->hmr0.s.fUsingDebugLoop \|\| pVmcsInfo->pShared->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
7582	("uVector=%#x u32XcptBitmap=%#X32\n",
7583	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
7584	NOREF(pVmcsInfo);
7585	# endif
7586	#endif
7587
7588	/*
7589	* Re-inject the exception into the guest. This cannot be a double-fault condition which
7590	* would have been handled while checking exits due to event delivery.
7591	*/
7592	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
7593
7594	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7595	int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
7596	AssertRCReturn(rc, rc);
7597	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%08RX64\n", uVector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7598	#endif
7599
7600	#ifdef VBOX_WITH_STATISTICS
7601	switch (uVector)
7602	{
7603	case X86_XCPT_DE: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE); break;
7604	case X86_XCPT_DB: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB); break;
7605	case X86_XCPT_BP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP); break;
7606	case X86_XCPT_OF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
7607	case X86_XCPT_BR: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBR); break;
7608	case X86_XCPT_UD: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestUD); break;
7609	case X86_XCPT_NM: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
7610	case X86_XCPT_DF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDF); break;
7611	case X86_XCPT_TS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestTS); break;
7612	case X86_XCPT_NP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestNP); break;
7613	case X86_XCPT_SS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestSS); break;
7614	case X86_XCPT_GP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP); break;
7615	case X86_XCPT_PF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF); break;
7616	case X86_XCPT_MF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF); break;
7617	case X86_XCPT_AC: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC); break;
7618	case X86_XCPT_XF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXF); break;
7619	default:
7620	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXcpUnk);
7621	break;
7622	}
7623	#endif
7624
7625	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
7626	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
7627	NOREF(uVector);
7628
7629	/* Re-inject the original exception into the guest. */
7630	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
7631	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
7632	return VINF_SUCCESS;
7633	}
7634
7635
7636	/**
7637	* VM-exit exception handler for all exceptions (except NMIs!).
7638	*
7639	* @remarks This may be called for both guests and nested-guests. Take care to not
7640	* make assumptions and avoid doing anything that is not relevant when
7641	* executing a nested-guest (e.g., Mesa driver hacks).
7642	*/
7643	static VBOXSTRICTRC vmxHCExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7644	{
7645	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
7646
7647	/*
7648	* If this VM-exit occurred while delivering an event through the guest IDT, take
7649	* action based on the return code and additional hints (e.g. for page-faults)
7650	* that will be updated in the VMX transient structure.
7651	*/
7652	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
7653	if (rcStrict == VINF_SUCCESS)
7654	{
7655	/*
7656	* If an exception caused a VM-exit due to delivery of an event, the original
7657	* event may have to be re-injected into the guest. We shall reinject it and
7658	* continue guest execution. However, page-fault is a complicated case and
7659	* needs additional processing done in vmxHCExitXcptPF().
7660	*/
7661	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
7662	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
7663	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
7664	\|\| uVector == X86_XCPT_PF)
7665	{
7666	switch (uVector)
7667	{
7668	case X86_XCPT_PF: return vmxHCExitXcptPF(pVCpu, pVmxTransient);
7669	case X86_XCPT_GP: return vmxHCExitXcptGP(pVCpu, pVmxTransient);
7670	case X86_XCPT_MF: return vmxHCExitXcptMF(pVCpu, pVmxTransient);
7671	case X86_XCPT_DB: return vmxHCExitXcptDB(pVCpu, pVmxTransient);
7672	case X86_XCPT_BP: return vmxHCExitXcptBP(pVCpu, pVmxTransient);
7673	case X86_XCPT_AC: return vmxHCExitXcptAC(pVCpu, pVmxTransient);
7674	case X86_XCPT_DE: return vmxHCExitXcptDE(pVCpu, pVmxTransient);
7675	default:
7676	return vmxHCExitXcptOthers(pVCpu, pVmxTransient);
7677	}
7678	}
7679	/* else: inject pending event before resuming guest execution. */
7680	}
7681	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
7682	{
7683	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
7684	rcStrict = VINF_SUCCESS;
7685	}
7686
7687	return rcStrict;
7688	}
7689	/** @} */
7690
7691
7692	/** @name VM-exit handlers.
7693	* @{
7694	*/
7695	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7696	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7697	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7698
7699	/**
7700	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7701	*/
7702	HMVMX_EXIT_DECL vmxHCExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7703	{
7704	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7705	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitExtInt);
7706
7707	#ifndef IN_NEM_DARWIN
7708	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
7709	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
7710	return VINF_SUCCESS;
7711	return VINF_EM_RAW_INTERRUPT;
7712	#else
7713	return VINF_SUCCESS;
7714	#endif
7715	}
7716
7717
7718	/**
7719	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
7720	* VM-exit.
7721	*/
7722	HMVMX_EXIT_DECL vmxHCExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7723	{
7724	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7725	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
7726
7727	vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
7728
7729	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
7730	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
7731	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
7732
7733	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7734	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
7735	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
7736	NOREF(pVmcsInfo);
7737
7738	VBOXSTRICTRC rcStrict;
7739	switch (uExitIntType)
7740	{
7741	#ifndef IN_NEM_DARWIN /* NMIs should never reach R3. */
7742	/*
7743	* Host physical NMIs:
7744	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
7745	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
7746	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
7747	*
7748	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
7749	* See Intel spec. 27.5.5 "Updating Non-Register State".
7750	*/
7751	case VMX_EXIT_INT_INFO_TYPE_NMI:
7752	{
7753	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
7754	break;
7755	}
7756	#endif
7757
7758	/*
7759	* Privileged software exceptions (#DB from ICEBP),
7760	* Software exceptions (#BP and #OF),
7761	* Hardware exceptions:
7762	* Process the required exceptions and resume guest execution if possible.
7763	*/
7764	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
7765	Assert(uVector == X86_XCPT_DB);
7766	RT_FALL_THRU();
7767	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
7768	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
7769	RT_FALL_THRU();
7770	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
7771	{
7772	NOREF(uVector);
7773	vmxHCReadToTransient< HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
7774	\| HMVMX_READ_EXIT_INSTR_LEN
7775	\| HMVMX_READ_IDT_VECTORING_INFO
7776	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
7777	rcStrict = vmxHCExitXcpt(pVCpu, pVmxTransient);
7778	break;
7779	}
7780
7781	default:
7782	{
7783	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
7784	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
7785	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
7786	break;
7787	}
7788	}
7789
7790	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
7791	return rcStrict;
7792	}
7793
7794
7795	/**
7796	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7797	*/
7798	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7799	{
7800	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7801
7802	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7803	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7804	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
7805
7806	/* Evaluate and deliver pending events and resume guest execution. */
7807	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIntWindow);
7808	return VINF_SUCCESS;
7809	}
7810
7811
7812	/**
7813	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7814	*/
7815	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7816	{
7817	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7818
7819	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7820	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
7821	{
7822	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7823	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
7824	}
7825
7826	Assert(!CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx));
7827
7828	/*
7829	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
7830	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
7831	*/
7832	uint32_t fIntrState;
7833	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
7834	AssertRC(rc);
7835	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7836	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7837	{
7838	CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
7839
7840	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7841	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
7842	AssertRC(rc);
7843	}
7844
7845	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready. */
7846	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
7847
7848	/* Evaluate and deliver pending events and resume guest execution. */
7849	return VINF_SUCCESS;
7850	}
7851
7852
7853	/**
7854	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7855	*/
7856	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7857	{
7858	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7859	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7860	}
7861
7862
7863	/**
7864	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7865	*/
7866	HMVMX_EXIT_NSRC_DECL vmxHCExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7867	{
7868	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7869	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7870	}
7871
7872
7873	/**
7874	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7875	*/
7876	HMVMX_EXIT_DECL vmxHCExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7877	{
7878	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7879
7880	/*
7881	* Get the state we need and update the exit history entry.
7882	*/
7883	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7884	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7885	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
7886	AssertRCReturn(rc, rc);
7887
7888	VBOXSTRICTRC rcStrict;
7889	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7890	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
7891	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7892	if (!pExitRec)
7893	{
7894	/*
7895	* Regular CPUID instruction execution.
7896	*/
7897	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
7898	if (rcStrict == VINF_SUCCESS)
7899	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7900	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7901	{
7902	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7903	rcStrict = VINF_SUCCESS;
7904	}
7905	}
7906	else
7907	{
7908	/*
7909	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7910	*/
7911	int rc2 = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
7912	IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
7913	AssertRCReturn(rc2, rc2);
7914
7915	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
7916	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
7917
7918	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7919	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
7920
7921	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7922	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7923	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7924	}
7925	return rcStrict;
7926	}
7927
7928
7929	/**
7930	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7931	*/
7932	HMVMX_EXIT_DECL vmxHCExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7933	{
7934	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7935
7936	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7937	int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CR4>(pVCpu, pVmcsInfo, __FUNCTION__);
7938	AssertRCReturn(rc, rc);
7939
7940	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
7941	return VINF_EM_RAW_EMULATE_INSTR;
7942
7943	AssertMsgFailed(("vmxHCExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
7944	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
7945	}
7946
7947
7948	/**
7949	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7950	*/
7951	HMVMX_EXIT_DECL vmxHCExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7952	{
7953	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7954
7955	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7956	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7957	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
7958	AssertRCReturn(rc, rc);
7959
7960	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
7961	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7962	{
7963	/* If we get a spurious VM-exit when TSC offsetting is enabled,
7964	we must reset offsetting on VM-entry. See @bugref{6634}. */
7965	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
7966	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7967	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7968	}
7969	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7970	{
7971	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7972	rcStrict = VINF_SUCCESS;
7973	}
7974	return rcStrict;
7975	}
7976
7977
7978	/**
7979	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7980	*/
7981	HMVMX_EXIT_DECL vmxHCExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7982	{
7983	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7984
7985	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7986	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
7987	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX>(pVCpu, pVmcsInfo, __FUNCTION__);
7988	AssertRCReturn(rc, rc);
7989
7990	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
7991	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7992	{
7993	/* If we get a spurious VM-exit when TSC offsetting is enabled,
7994	we must reset offsetting on VM-reentry. See @bugref{6634}. */
7995	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
7996	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7997	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7998	}
7999	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8000	{
8001	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8002	rcStrict = VINF_SUCCESS;
8003	}
8004	return rcStrict;
8005	}
8006
8007
8008	/**
8009	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8010	*/
8011	HMVMX_EXIT_DECL vmxHCExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8012	{
8013	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8014
8015	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8016	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8017	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4>(pVCpu, pVmcsInfo, __FUNCTION__);
8018	AssertRCReturn(rc, rc);
8019
8020	VBOXSTRICTRC rcStrict = IEMExecDecodedRdpmc(pVCpu, pVmxTransient->cbExitInstr);
8021	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8022	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
8023	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8024	{
8025	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8026	rcStrict = VINF_SUCCESS;
8027	}
8028	return rcStrict;
8029	}
8030
8031
8032	/**
8033	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
8034	*/
8035	HMVMX_EXIT_DECL vmxHCExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8036	{
8037	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8038
8039	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
8040	if (EMAreHypercallInstructionsEnabled(pVCpu))
8041	{
8042	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8043	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RIP
8044	\| CPUMCTX_EXTRN_RFLAGS
8045	\| CPUMCTX_EXTRN_CR0
8046	\| CPUMCTX_EXTRN_SS
8047	\| CPUMCTX_EXTRN_CS
8048	\| CPUMCTX_EXTRN_EFER>(pVCpu, pVmcsInfo, __FUNCTION__);
8049	AssertRCReturn(rc, rc);
8050
8051	/* Perform the hypercall. */
8052	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8053	if (rcStrict == VINF_SUCCESS)
8054	{
8055	rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
8056	AssertRCReturn(rc, rc);
8057	}
8058	else
8059	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
8060	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
8061	\|\| RT_FAILURE(rcStrict));
8062
8063	/* If the hypercall changes anything other than guest's general-purpose registers,
8064	we would need to reload the guest changed bits here before VM-entry. */
8065	}
8066	else
8067	Log4Func(("Hypercalls not enabled\n"));
8068
8069	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
8070	if (RT_FAILURE(rcStrict))
8071	{
8072	vmxHCSetPendingXcptUD(pVCpu);
8073	rcStrict = VINF_SUCCESS;
8074	}
8075
8076	return rcStrict;
8077	}
8078
8079
8080	/**
8081	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8082	*/
8083	HMVMX_EXIT_DECL vmxHCExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8084	{
8085	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8086	#ifndef IN_NEM_DARWIN
8087	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| pVCpu->hmr0.s.fUsingDebugLoop);
8088	#endif
8089
8090	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8091	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8092	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8093	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
8094	AssertRCReturn(rc, rc);
8095
8096	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
8097
8098	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
8099	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
8100	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8101	{
8102	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8103	rcStrict = VINF_SUCCESS;
8104	}
8105	else
8106	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
8107	VBOXSTRICTRC_VAL(rcStrict)));
8108	return rcStrict;
8109	}
8110
8111
8112	/**
8113	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8114	*/
8115	HMVMX_EXIT_DECL vmxHCExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8116	{
8117	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8118
8119	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8120	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8121	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS>(pVCpu, pVmcsInfo, __FUNCTION__);
8122	AssertRCReturn(rc, rc);
8123
8124	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
8125	if (rcStrict == VINF_SUCCESS)
8126	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
8127	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8128	{
8129	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8130	rcStrict = VINF_SUCCESS;
8131	}
8132
8133	return rcStrict;
8134	}
8135
8136
8137	/**
8138	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8139	*/
8140	HMVMX_EXIT_DECL vmxHCExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8141	{
8142	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8143
8144	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8145	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8146	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
8147	AssertRCReturn(rc, rc);
8148
8149	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
8150	if (RT_SUCCESS(rcStrict))
8151	{
8152	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
8153	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
8154	rcStrict = VINF_SUCCESS;
8155	}
8156
8157	return rcStrict;
8158	}
8159
8160
8161	/**
8162	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
8163	* VM-exit.
8164	*/
8165	HMVMX_EXIT_DECL vmxHCExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8166	{
8167	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8168	return VINF_EM_RESET;
8169	}
8170
8171
8172	/**
8173	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
8174	*/
8175	HMVMX_EXIT_DECL vmxHCExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8176	{
8177	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8178
8179	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
8180	AssertRCReturn(rc, rc);
8181
8182	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
8183	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
8184	rc = VINF_SUCCESS;
8185	else
8186	rc = VINF_EM_HALT;
8187
8188	if (rc != VINF_SUCCESS)
8189	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHltToR3);
8190	return rc;
8191	}
8192
8193
8194	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
8195	/**
8196	* VM-exit handler for instructions that result in a \#UD exception delivered to
8197	* the guest.
8198	*/
8199	HMVMX_EXIT_NSRC_DECL vmxHCExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8200	{
8201	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8202	vmxHCSetPendingXcptUD(pVCpu);
8203	return VINF_SUCCESS;
8204	}
8205	#endif
8206
8207
8208	/**
8209	* VM-exit handler for expiry of the VMX-preemption timer.
8210	*/
8211	HMVMX_EXIT_DECL vmxHCExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8212	{
8213	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8214
8215	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
8216	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
8217	Log12(("vmxHCExitPreemptTimer:\n"));
8218
8219	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
8220	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8221	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
8222	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitPreemptTimer);
8223	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
8224	}
8225
8226
8227	/**
8228	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
8229	*/
8230	HMVMX_EXIT_DECL vmxHCExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8231	{
8232	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8233
8234	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8235	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8236	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4>(pVCpu, pVmcsInfo, __FUNCTION__);
8237	AssertRCReturn(rc, rc);
8238
8239	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
8240	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8241	: HM_CHANGED_RAISED_XCPT_MASK);
8242
8243	#ifndef IN_NEM_DARWIN
8244	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8245	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
8246	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
8247	{
8248	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
8249	hmR0VmxUpdateStartVmFunction(pVCpu);
8250	}
8251	#endif
8252
8253	return rcStrict;
8254	}
8255
8256
8257	/**
8258	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
8259	*/
8260	HMVMX_EXIT_DECL vmxHCExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8261	{
8262	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8263
8264	/** @todo Enable the new code after finding a reliably guest test-case. */
8265	#if 1
8266	return VERR_EM_INTERPRETER;
8267	#else
8268	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8269	\| HMVMX_READ_EXIT_INSTR_INFO
8270	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8271	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
8272	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
8273	AssertRCReturn(rc, rc);
8274
8275	/* Paranoia. Ensure this has a memory operand. */
8276	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
8277
8278	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
8279	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
8280	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
8281	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
8282
8283	RTGCPTR GCPtrDesc;
8284	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
8285
8286	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
8287	GCPtrDesc, uType);
8288	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8289	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
8290	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8291	{
8292	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8293	rcStrict = VINF_SUCCESS;
8294	}
8295	return rcStrict;
8296	#endif
8297	}
8298
8299
8300	/**
8301	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
8302	* VM-exit.
8303	*/
8304	HMVMX_EXIT_NSRC_DECL vmxHCExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8305	{
8306	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8307	int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8308	AssertRCReturn(rc, rc);
8309
8310	rc = vmxHCCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
8311	if (RT_FAILURE(rc))
8312	return rc;
8313
8314	uint32_t const uInvalidReason = vmxHCCheckGuestState(pVCpu, pVmcsInfo);
8315	NOREF(uInvalidReason);
8316
8317	#ifdef VBOX_STRICT
8318	uint32_t fIntrState;
8319	uint64_t u64Val;
8320	vmxHCReadToTransient< HMVMX_READ_EXIT_INSTR_INFO
8321	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8322	vmxHCReadEntryXcptErrorCodeVmcs(pVCpu, pVmxTransient);
8323
8324	Log4(("uInvalidReason %u\n", uInvalidReason));
8325	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
8326	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
8327	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
8328
8329	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
8330	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
8331	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
8332	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
8333	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
8334	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
8335	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
8336	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
8337	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
8338	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
8339	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
8340	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
8341	# ifndef IN_NEM_DARWIN
8342	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
8343	{
8344	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
8345	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
8346	}
8347
8348	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
8349	# endif
8350	#endif
8351
8352	return VERR_VMX_INVALID_GUEST_STATE;
8353	}
8354
8355	/**
8356	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
8357	*/
8358	HMVMX_EXIT_NSRC_DECL vmxHCExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8359	{
8360	/*
8361	* Cumulative notes of all recognized but unexpected VM-exits.
8362	*
8363	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
8364	* nested-paging is used.
8365	*
8366	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
8367	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
8368	* this function (and thereby stop VM execution) for handling such instructions.
8369	*
8370	*
8371	* VMX_EXIT_INIT_SIGNAL:
8372	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
8373	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
8374	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
8375	*
8376	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
8377	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
8378	* See Intel spec. "23.8 Restrictions on VMX operation".
8379	*
8380	* VMX_EXIT_SIPI:
8381	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
8382	* activity state is used. We don't make use of it as our guests don't have direct
8383	* access to the host local APIC.
8384	*
8385	* See Intel spec. 25.3 "Other Causes of VM-exits".
8386	*
8387	* VMX_EXIT_IO_SMI:
8388	* VMX_EXIT_SMI:
8389	* This can only happen if we support dual-monitor treatment of SMI, which can be
8390	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
8391	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
8392	* VMX root mode or receive an SMI. If we get here, something funny is going on.
8393	*
8394	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
8395	* See Intel spec. 25.3 "Other Causes of VM-Exits"
8396	*
8397	* VMX_EXIT_ERR_MSR_LOAD:
8398	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
8399	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
8400	* execution.
8401	*
8402	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
8403	*
8404	* VMX_EXIT_ERR_MACHINE_CHECK:
8405	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
8406	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
8407	* #MC exception abort class exception is raised. We thus cannot assume a
8408	* reasonable chance of continuing any sort of execution and we bail.
8409	*
8410	* See Intel spec. 15.1 "Machine-check Architecture".
8411	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
8412	*
8413	* VMX_EXIT_PML_FULL:
8414	* VMX_EXIT_VIRTUALIZED_EOI:
8415	* VMX_EXIT_APIC_WRITE:
8416	* We do not currently support any of these features and thus they are all unexpected
8417	* VM-exits.
8418	*
8419	* VMX_EXIT_GDTR_IDTR_ACCESS:
8420	* VMX_EXIT_LDTR_TR_ACCESS:
8421	* VMX_EXIT_RDRAND:
8422	* VMX_EXIT_RSM:
8423	* VMX_EXIT_VMFUNC:
8424	* VMX_EXIT_ENCLS:
8425	* VMX_EXIT_RDSEED:
8426	* VMX_EXIT_XSAVES:
8427	* VMX_EXIT_XRSTORS:
8428	* VMX_EXIT_UMWAIT:
8429	* VMX_EXIT_TPAUSE:
8430	* VMX_EXIT_LOADIWKEY:
8431	* These VM-exits are -not- caused unconditionally by execution of the corresponding
8432	* instruction. Any VM-exit for these instructions indicate a hardware problem,
8433	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
8434	*
8435	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
8436	*/
8437	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8438	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
8439	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
8440	}
8441
8442
8443	/**
8444	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
8445	*/
8446	HMVMX_EXIT_DECL vmxHCExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8447	{
8448	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8449
8450	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8451
8452	/** @todo Optimize this: We currently drag in the whole MSR state
8453	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
8454	* MSRs required. That would require changes to IEM and possibly CPUM too.
8455	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
8456	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8457	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
8458	int rc;
8459	switch (idMsr)
8460	{
8461	default:
8462	rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS>(pVCpu, pVmcsInfo,
8463	__FUNCTION__);
8464	AssertRCReturn(rc, rc);
8465	break;
8466	case MSR_K8_FS_BASE:
8467	rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS
8468	\| CPUMCTX_EXTRN_FS>(pVCpu, pVmcsInfo, __FUNCTION__);
8469	AssertRCReturn(rc, rc);
8470	break;
8471	case MSR_K8_GS_BASE:
8472	rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS
8473	\| CPUMCTX_EXTRN_GS>(pVCpu, pVmcsInfo, __FUNCTION__);
8474	AssertRCReturn(rc, rc);
8475	break;
8476	}
8477
8478	Log4Func(("ecx=%#RX32\n", idMsr));
8479
8480	#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
8481	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
8482	{
8483	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
8484	&& idMsr != MSR_K6_EFER)
8485	{
8486	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
8487	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
8488	}
8489	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
8490	{
8491	Assert(pVmcsInfo->pvMsrBitmap);
8492	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
8493	if (fMsrpm & VMXMSRPM_ALLOW_RD)
8494	{
8495	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
8496	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
8497	}
8498	}
8499	}
8500	#endif
8501
8502	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
8503	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitRdmsr);
8504	if (rcStrict == VINF_SUCCESS)
8505	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
8506	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8507	{
8508	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8509	rcStrict = VINF_SUCCESS;
8510	}
8511	else
8512	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
8513	("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
8514
8515	return rcStrict;
8516	}
8517
8518
8519	/**
8520	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
8521	*/
8522	HMVMX_EXIT_DECL vmxHCExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8523	{
8524	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8525
8526	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8527
8528	/*
8529	* The FS and GS base MSRs are not part of the above all-MSRs mask.
8530	* Although we don't need to fetch the base as it will be overwritten shortly, while
8531	* loading guest-state we would also load the entire segment register including limit
8532	* and attributes and thus we need to load them here.
8533	*/
8534	/** @todo Optimize this: We currently drag in the whole MSR state
8535	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
8536	* MSRs required. That would require changes to IEM and possibly CPUM too.
8537	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
8538	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8539	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
8540	int rc;
8541	switch (idMsr)
8542	{
8543	default:
8544	rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS>(pVCpu, pVmcsInfo,
8545	__FUNCTION__);
8546	AssertRCReturn(rc, rc);
8547	break;
8548
8549	case MSR_K8_FS_BASE:
8550	rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS
8551	\| CPUMCTX_EXTRN_FS>(pVCpu, pVmcsInfo, __FUNCTION__);
8552	AssertRCReturn(rc, rc);
8553	break;
8554	case MSR_K8_GS_BASE:
8555	rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS
8556	\| CPUMCTX_EXTRN_GS>(pVCpu, pVmcsInfo, __FUNCTION__);
8557	AssertRCReturn(rc, rc);
8558	break;
8559	}
8560	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
8561
8562	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
8563	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitWrmsr);
8564
8565	if (rcStrict == VINF_SUCCESS)
8566	{
8567	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
8568
8569	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
8570	if ( idMsr == MSR_IA32_APICBASE
8571	\|\| ( idMsr >= MSR_IA32_X2APIC_START
8572	&& idMsr <= MSR_IA32_X2APIC_END))
8573	{
8574	/*
8575	* We've already saved the APIC related guest-state (TPR) in post-run phase.
8576	* When full APIC register virtualization is implemented we'll have to make
8577	* sure APIC state is saved from the VMCS before IEM changes it.
8578	*/
8579	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8580	}
8581	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
8582	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
8583	else if (idMsr == MSR_K6_EFER)
8584	{
8585	/*
8586	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
8587	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
8588	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
8589	*/
8590	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
8591	}
8592
8593	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
8594	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8595	{
8596	switch (idMsr)
8597	{
8598	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
8599	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
8600	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
8601	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_FS); break;
8602	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_GS); break;
8603	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
8604	default:
8605	{
8606	#ifndef IN_NEM_DARWIN
8607	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
8608	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
8609	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
8610	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
8611	#else
8612	AssertMsgFailed(("TODO\n"));
8613	#endif
8614	break;
8615	}
8616	}
8617	}
8618	#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
8619	else
8620	{
8621	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
8622	switch (idMsr)
8623	{
8624	case MSR_IA32_SYSENTER_CS:
8625	case MSR_IA32_SYSENTER_EIP:
8626	case MSR_IA32_SYSENTER_ESP:
8627	case MSR_K8_FS_BASE:
8628	case MSR_K8_GS_BASE:
8629	{
8630	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
8631	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
8632	}
8633
8634	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
8635	default:
8636	{
8637	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
8638	{
8639	/* EFER MSR writes are always intercepted. */
8640	if (idMsr != MSR_K6_EFER)
8641	{
8642	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
8643	idMsr));
8644	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
8645	}
8646	}
8647
8648	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
8649	{
8650	Assert(pVmcsInfo->pvMsrBitmap);
8651	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
8652	if (fMsrpm & VMXMSRPM_ALLOW_WR)
8653	{
8654	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
8655	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
8656	}
8657	}
8658	break;
8659	}
8660	}
8661	}
8662	#endif /* VBOX_STRICT */
8663	}
8664	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8665	{
8666	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8667	rcStrict = VINF_SUCCESS;
8668	}
8669	else
8670	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
8671	("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
8672
8673	return rcStrict;
8674	}
8675
8676
8677	/**
8678	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8679	*/
8680	HMVMX_EXIT_DECL vmxHCExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8681	{
8682	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8683
8684	/** @todo The guest has likely hit a contended spinlock. We might want to
8685	* poke a schedule different guest VCPU. */
8686	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
8687	if (RT_SUCCESS(rc))
8688	return VINF_EM_RAW_INTERRUPT;
8689
8690	AssertMsgFailed(("vmxHCExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
8691	return rc;
8692	}
8693
8694
8695	/**
8696	* VM-exit handler for when the TPR value is lowered below the specified
8697	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8698	*/
8699	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8700	{
8701	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8702	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
8703
8704	/*
8705	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
8706	* We'll re-evaluate pending interrupts and inject them before the next VM
8707	* entry so we can just continue execution here.
8708	*/
8709	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTprBelowThreshold);
8710	return VINF_SUCCESS;
8711	}
8712
8713
8714	/**
8715	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8716	* VM-exit.
8717	*
8718	* @retval VINF_SUCCESS when guest execution can continue.
8719	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8720	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
8721	* incompatible guest state for VMX execution (real-on-v86 case).
8722	*/
8723	HMVMX_EXIT_DECL vmxHCExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8724	{
8725	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8726	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
8727
8728	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8729	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8730	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8731
8732	VBOXSTRICTRC rcStrict;
8733	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8734	uint64_t const uExitQual = pVmxTransient->uExitQual;
8735	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
8736	switch (uAccessType)
8737	{
8738	/*
8739	* MOV to CRx.
8740	*/
8741	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
8742	{
8743	/*
8744	* When PAE paging is used, the CPU will reload PAE PDPTEs from CR3 when the guest
8745	* changes certain bits even in CR0, CR4 (and not just CR3). We are currently fine
8746	* since IEM_CPUMCTX_EXTRN_MUST_MASK (used below) includes CR3 which will import
8747	* PAE PDPTEs as well.
8748	*/
8749	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
8750	AssertRCReturn(rc, rc);
8751
8752	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
8753	#ifndef IN_NEM_DARWIN
8754	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
8755	#endif
8756	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
8757	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
8758
8759	/*
8760	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
8761	* - When nested paging isn't used.
8762	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
8763	* - We are executing in the VM debug loop.
8764	*/
8765	#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
8766	# ifndef IN_NEM_DARWIN
8767	Assert( iCrReg != 3
8768	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
8769	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
8770	\|\| pVCpu->hmr0.s.fUsingDebugLoop);
8771	# else
8772	Assert( iCrReg != 3
8773	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
8774	# endif
8775	#endif
8776
8777	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
8778	Assert( iCrReg != 8
8779	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
8780
8781	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
8782	AssertMsg( rcStrict == VINF_SUCCESS
8783	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8784
8785	#ifndef IN_NEM_DARWIN
8786	/*
8787	* This is a kludge for handling switches back to real mode when we try to use
8788	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
8789	* deal with special selector values, so we have to return to ring-3 and run
8790	* there till the selector values are V86 mode compatible.
8791	*
8792	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
8793	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
8794	* this function.
8795	*/
8796	if ( iCrReg == 0
8797	&& rcStrict == VINF_SUCCESS
8798	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
8799	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
8800	&& (uOldCr0 & X86_CR0_PE)
8801	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
8802	{
8803	/** @todo Check selectors rather than returning all the time. */
8804	Assert(!pVmxTransient->fIsNestedGuest);
8805	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
8806	rcStrict = VINF_EM_RESCHEDULE_REM;
8807	}
8808	#endif
8809
8810	break;
8811	}
8812
8813	/*
8814	* MOV from CRx.
8815	*/
8816	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
8817	{
8818	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
8819	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
8820
8821	/*
8822	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
8823	* - When nested paging isn't used.
8824	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
8825	* - We are executing in the VM debug loop.
8826	*/
8827	#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
8828	# ifndef IN_NEM_DARWIN
8829	Assert( iCrReg != 3
8830	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
8831	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
8832	\|\| pVCpu->hmr0.s.fLeaveDone);
8833	# else
8834	Assert( iCrReg != 3
8835	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
8836	# endif
8837	#endif
8838
8839	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8840	Assert( iCrReg != 8
8841	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
8842
8843	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
8844	break;
8845	}
8846
8847	/*
8848	* CLTS (Clear Task-Switch Flag in CR0).
8849	*/
8850	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
8851	{
8852	rcStrict = vmxHCExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
8853	break;
8854	}
8855
8856	/*
8857	* LMSW (Load Machine-Status Word into CR0).
8858	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
8859	*/
8860	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
8861	{
8862	RTGCPTR GCPtrEffDst;
8863	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
8864	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
8865	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
8866	if (fMemOperand)
8867	{
8868	vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
8869	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
8870	}
8871	else
8872	GCPtrEffDst = NIL_RTGCPTR;
8873	rcStrict = vmxHCExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
8874	break;
8875	}
8876
8877	default:
8878	{
8879	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
8880	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
8881	}
8882	}
8883
8884	Assert((VCPU_2_VMXSTATE(pVCpu).fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
8885	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
8886	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
8887
8888	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
8889	NOREF(pVM);
8890	return rcStrict;
8891	}
8892
8893
8894	/**
8895	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8896	* VM-exit.
8897	*/
8898	HMVMX_EXIT_DECL vmxHCExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8899	{
8900	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8901	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
8902
8903	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8904	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8905	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
8906	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
8907	#define VMX_HC_EXIT_IO_INSTR_INITIAL_REGS (IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER)
8908	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8909	int rc = vmxHCImportGuestState<VMX_HC_EXIT_IO_INSTR_INITIAL_REGS>(pVCpu, pVmcsInfo, __FUNCTION__);
8910	AssertRCReturn(rc, rc);
8911
8912	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8913	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
8914	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
8915	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
8916	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
8917	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
8918	bool const fDbgStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
8919	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
8920
8921	/*
8922	* Update exit history to see if this exit can be optimized.
8923	*/
8924	VBOXSTRICTRC rcStrict;
8925	PCEMEXITREC pExitRec = NULL;
8926	if ( !fGstStepping
8927	&& !fDbgStepping)
8928	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
8929	!fIOString
8930	? !fIOWrite
8931	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
8932	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
8933	: !fIOWrite
8934	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
8935	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
8936	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
8937	if (!pExitRec)
8938	{
8939	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
8940	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
8941
8942	uint32_t const cbValue = s_aIOSizes[uIOSize];
8943	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
8944	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
8945	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8946	if (fIOString)
8947	{
8948	/*
8949	* INS/OUTS - I/O String instruction.
8950	*
8951	* Use instruction-information if available, otherwise fall back on
8952	* interpreting the instruction.
8953	*/
8954	Log4Func(("cs:rip=%#04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
8955	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
8956	bool const fInsOutsInfo = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
8957	if (fInsOutsInfo)
8958	{
8959	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
8960	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
8961	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
8962	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
8963	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
8964	if (fIOWrite)
8965	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
8966	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
8967	else
8968	{
8969	/*
8970	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
8971	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
8972	* See Intel Instruction spec. for "INS".
8973	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
8974	*/
8975	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
8976	}
8977	}
8978	else
8979	rcStrict = IEMExecOne(pVCpu);
8980
8981	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
8982	fUpdateRipAlready = true;
8983	}
8984	else
8985	{
8986	/*
8987	* IN/OUT - I/O instruction.
8988	*/
8989	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
8990	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
8991	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
8992	if (fIOWrite)
8993	{
8994	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
8995	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite);
8996	#ifndef IN_NEM_DARWIN
8997	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
8998	&& !pCtx->eflags.Bits.u1TF)
8999	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
9000	#endif
9001	}
9002	else
9003	{
9004	rcStrict = VERR_GCM_NOT_HANDLED;
9005	if (GCMIsInterceptingIOPortRead(pVCpu, uIOPort, cbValue))
9006	{
9007	rcStrict = GCMInterceptedIOPortRead(pVCpu, pCtx, uIOPort, cbValue);
9008	if (rcStrict == VINF_GCM_HANDLED_ADVANCE_RIP \|\| rcStrict == VINF_GCM_HANDLED)
9009	{
9010	/* ASSUMES we don't need to update fCtxChanged when regular GPRs change here. */
9011	fUpdateRipAlready = rcStrict == VINF_GCM_HANDLED;
9012	if (rcStrict == VINF_GCM_HANDLED)
9013	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
9014	rcStrict = VINF_SUCCESS;
9015	}
9016	else
9017	Assert(rcStrict == VERR_GCM_NOT_HANDLED);
9018	}
9019
9020	if (rcStrict == VERR_GCM_NOT_HANDLED)
9021	{
9022	uint32_t u32Result = 0;
9023	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9024	if (IOM_SUCCESS(rcStrict))
9025	{
9026	/* Save result of I/O IN instr. in AL/AX/EAX. */
9027	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
9028	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9029	}
9030	#ifndef IN_NEM_DARWIN
9031	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
9032	&& !pCtx->eflags.Bits.u1TF)
9033	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
9034	#endif
9035	}
9036	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIORead);
9037	}
9038	}
9039
9040	if (IOM_SUCCESS(rcStrict))
9041	{
9042	if (!fUpdateRipAlready)
9043	{
9044	vmxHCAdvanceGuestRipBy(pVCpu, cbInstr);
9045	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
9046	}
9047
9048	/*
9049	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
9050	* while booting Fedora 17 64-bit guest.
9051	*
9052	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
9053	*/
9054	if (fIOString)
9055	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
9056
9057	/*
9058	* If any I/O breakpoints are armed, we need to check if one triggered
9059	* and take appropriate action.
9060	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9061	*/
9062	#if 1
9063	AssertCompile(VMX_HC_EXIT_IO_INSTR_INITIAL_REGS & CPUMCTX_EXTRN_DR7);
9064	#else
9065	AssertCompile(!(VMX_HC_EXIT_IO_INSTR_INITIAL_REGS & CPUMCTX_EXTRN_DR7));
9066	rc = vmxHCImportGuestState<CPUMCTX_EXTRN_DR7>(pVCpu, pVmcsInfo);
9067	AssertRCReturn(rc, rc);
9068	#endif
9069
9070	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9071	* execution engines about whether hyper BPs and such are pending. */
9072	uint32_t const uDr7 = pCtx->dr[7];
9073	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9074	&& X86_DR7_ANY_RW_IO(uDr7)
9075	&& (pCtx->cr4 & X86_CR4_DE))
9076	\|\| DBGFBpIsHwIoArmed(pVM)))
9077	{
9078	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxIoCheck);
9079
9080	#ifndef IN_NEM_DARWIN
9081	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
9082	VMMRZCallRing3Disable(pVCpu);
9083	HM_DISABLE_PREEMPT(pVCpu);
9084
9085	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
9086
9087	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
9088	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9089	{
9090	/* Raise #DB. */
9091	if (fIsGuestDbgActive)
9092	ASMSetDR6(pCtx->dr[6]);
9093	if (pCtx->dr[7] != uDr7)
9094	VCPU_2_VMXSTATE(pVCpu).fCtxChanged \|= HM_CHANGED_GUEST_DR7;
9095
9096	vmxHCSetPendingXcptDB(pVCpu);
9097	}
9098	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
9099	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
9100	else if ( rcStrict2 != VINF_SUCCESS
9101	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9102	rcStrict = rcStrict2;
9103	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
9104
9105	HM_RESTORE_PREEMPT();
9106	VMMRZCallRing3Enable(pVCpu);
9107	#else
9108	/** @todo */
9109	#endif
9110	}
9111	}
9112
9113	#ifdef VBOX_STRICT
9114	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
9115	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
9116	Assert(!fIOWrite);
9117	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
9118	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
9119	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
9120	Assert(fIOWrite);
9121	else
9122	{
9123	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9124	* statuses, that the VMM device and some others may return. See
9125	* IOM_SUCCESS() for guidance. */
9126	AssertMsg( RT_FAILURE(rcStrict)
9127	\|\| rcStrict == VINF_SUCCESS
9128	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9129	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9130	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9131	\|\| rcStrict == VINF_EM_RAW_TO_R3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9132	# endif
9133	}
9134	#endif
9135	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
9136	}
9137	else
9138	{
9139	/*
9140	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
9141	*/
9142	int rc2 = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
9143	VMX_HC_EXIT_IO_INSTR_INITIAL_REGS>(pVCpu, pVmcsInfo, __FUNCTION__);
9144	AssertRCReturn(rc2, rc2);
9145	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIORead
9146	: fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringRead);
9147	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
9148	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9149	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
9150	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
9151
9152	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
9153	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9154
9155	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
9156	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9157	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
9158	}
9159	return rcStrict;
9160	}
9161
9162
9163	/**
9164	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9165	* VM-exit.
9166	*/
9167	HMVMX_EXIT_DECL vmxHCExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9168	{
9169	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9170
9171	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9172	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
9173	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
9174	{
9175	vmxHCReadToTransient<HMVMX_READ_IDT_VECTORING_INFO>(pVCpu, pVmxTransient);
9176	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
9177	{
9178	uint32_t uErrCode;
9179	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
9180	{
9181	vmxHCReadToTransient<HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
9182	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
9183	}
9184	else
9185	uErrCode = 0;
9186
9187	RTGCUINTPTR GCPtrFaultAddress;
9188	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
9189	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
9190	else
9191	GCPtrFaultAddress = 0;
9192
9193	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9194
9195	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
9196	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
9197
9198	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
9199	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
9200	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
9201	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9202	}
9203	}
9204
9205	/* Fall back to the interpreter to emulate the task-switch. */
9206	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
9207	return VERR_EM_INTERPRETER;
9208	}
9209
9210
9211	/**
9212	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9213	*/
9214	HMVMX_EXIT_DECL vmxHCExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9215	{
9216	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9217
9218	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9219	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
9220	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
9221	AssertRC(rc);
9222	return VINF_EM_DBG_STEPPED;
9223	}
9224
9225
9226	/**
9227	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9228	*/
9229	HMVMX_EXIT_DECL vmxHCExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9230	{
9231	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9232	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitApicAccess);
9233
9234	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9235	\| HMVMX_READ_EXIT_INSTR_LEN
9236	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
9237	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
9238	\| HMVMX_READ_IDT_VECTORING_INFO
9239	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
9240
9241	/*
9242	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
9243	*/
9244	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
9245	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9246	{
9247	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
9248	if (RT_UNLIKELY(VCPU_2_VMXSTATE(pVCpu).Event.fPending))
9249	{
9250	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
9251	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9252	}
9253	}
9254	else
9255	{
9256	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
9257	return rcStrict;
9258	}
9259
9260	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
9261	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9262	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
9263	AssertRCReturn(rc, rc);
9264
9265	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9266	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
9267	switch (uAccessType)
9268	{
9269	#ifndef IN_NEM_DARWIN
9270	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9271	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9272	{
9273	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
9274	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
9275	("vmxHCExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9276
9277	RTGCPHYS GCPhys = VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
9278	GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
9279	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
9280	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
9281	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
9282
9283	rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
9284	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
9285	Log4Func(("IOMR0MmioPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9286	if ( rcStrict == VINF_SUCCESS
9287	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
9288	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
9289	{
9290	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9291	\| HM_CHANGED_GUEST_APIC_TPR);
9292	rcStrict = VINF_SUCCESS;
9293	}
9294	break;
9295	}
9296	#else
9297	/** @todo */
9298	#endif
9299
9300	default:
9301	{
9302	Log4Func(("uAccessType=%#x\n", uAccessType));
9303	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9304	break;
9305	}
9306	}
9307
9308	if (rcStrict != VINF_SUCCESS)
9309	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchApicAccessToR3);
9310	return rcStrict;
9311	}
9312
9313
9314	/**
9315	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9316	* VM-exit.
9317	*/
9318	HMVMX_EXIT_DECL vmxHCExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9319	{
9320	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9321	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9322
9323	/*
9324	* We might also get this VM-exit if the nested-guest isn't intercepting MOV DRx accesses.
9325	* In such a case, rather than disabling MOV DRx intercepts and resuming execution, we
9326	* must emulate the MOV DRx access.
9327	*/
9328	if (!pVmxTransient->fIsNestedGuest)
9329	{
9330	/* We should -not- get this VM-exit if the guest's debug registers were active. */
9331	if ( pVmxTransient->fWasGuestDebugStateActive
9332	#ifdef VMX_WITH_MAYBE_ALWAYS_INTERCEPT_MOV_DRX
9333	&& !pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fAlwaysInterceptMovDRx
9334	#endif
9335	)
9336	{
9337	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
9338	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
9339	}
9340
9341	if ( !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction
9342	&& !pVmxTransient->fWasHyperDebugStateActive)
9343	{
9344	Assert(!DBGFIsStepping(pVCpu));
9345	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
9346
9347	/* Whether we disable intercepting MOV DRx instructions and resume
9348	the current one, or emulate it and keep intercepting them is
9349	configurable. Though it usually comes down to whether there are
9350	any new DR6 & DR7 bits (RTM) we want to hide from the guest. */
9351	#ifdef VMX_WITH_MAYBE_ALWAYS_INTERCEPT_MOV_DRX
9352	bool const fResumeInstruction = !pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fAlwaysInterceptMovDRx;
9353	#else
9354	bool const fResumeInstruction = true;
9355	#endif
9356	if (fResumeInstruction)
9357	{
9358	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
9359	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
9360	AssertRC(rc);
9361	}
9362
9363	#ifndef IN_NEM_DARWIN
9364	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
9365	VMMRZCallRing3Disable(pVCpu);
9366	HM_DISABLE_PREEMPT(pVCpu);
9367
9368	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9369	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9370	Assert(CPUMIsGuestDebugStateActive(pVCpu));
9371
9372	HM_RESTORE_PREEMPT();
9373	VMMRZCallRing3Enable(pVCpu);
9374	#else
9375	CPUMR3NemActivateGuestDebugState(pVCpu);
9376	Assert(CPUMIsGuestDebugStateActive(pVCpu));
9377	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
9378	#endif
9379
9380	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxContextSwitch);
9381	if (fResumeInstruction)
9382	{
9383	#ifdef VBOX_WITH_STATISTICS
9384	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
9385	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
9386	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
9387	else
9388	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
9389	#endif
9390	return VINF_SUCCESS;
9391	}
9392	}
9393	}
9394
9395	/*
9396	* Import state. We must have DR7 loaded here as it's always consulted,
9397	* both for reading and writing. The other debug registers are never
9398	* exported as such.
9399	*/
9400	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION \| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9401	int rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
9402	\| CPUMCTX_EXTRN_GPRS_MASK
9403	\| CPUMCTX_EXTRN_DR7>(pVCpu, pVmcsInfo, __FUNCTION__);
9404	AssertRCReturn(rc, rc);
9405
9406	uint8_t const iGReg = VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual);
9407	uint8_t const iDrReg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9408	Log4Func(("cs:rip=%#04x:%08RX64 r%d %s dr%d\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, iGReg,
9409	VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE ? "->" : "<-", iDrReg));
9410
9411	VBOXSTRICTRC rcStrict;
9412	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
9413	{
9414	/*
9415	* Write DRx register.
9416	*/
9417	rcStrict = IEMExecDecodedMovDRxWrite(pVCpu, pVmxTransient->cbExitInstr, iDrReg, iGReg);
9418	AssertMsg( rcStrict == VINF_SUCCESS
9419	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9420
9421	if (rcStrict == VINF_SUCCESS)
9422	{
9423	/** @todo r=bird: Not sure why we always flag DR7 as modified here, but I've
9424	* kept it for now to avoid breaking something non-obvious. */
9425	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
9426	\| HM_CHANGED_GUEST_DR7);
9427	/* Update the DR6 register if guest debug state is active, otherwise we'll
9428	trash it when calling CPUMR0DebugStateMaybeSaveGuestAndRestoreHost. */
9429	if (iDrReg == 6 && CPUMIsGuestDebugStateActive(pVCpu))
9430	ASMSetDR6(pVCpu->cpum.GstCtx.dr[6]);
9431	Log4Func(("r%d=%#RX64 => dr%d=%#RX64\n", iGReg, pVCpu->cpum.GstCtx.aGRegs[iGReg].u,
9432	iDrReg, pVCpu->cpum.GstCtx.dr[iDrReg]));
9433	}
9434	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9435	{
9436	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9437	rcStrict = VINF_SUCCESS;
9438	}
9439
9440	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
9441	}
9442	else
9443	{
9444	/*
9445	* Read DRx register into a general purpose register.
9446	*/
9447	rcStrict = IEMExecDecodedMovDRxRead(pVCpu, pVmxTransient->cbExitInstr, iGReg, iDrReg);
9448	AssertMsg( rcStrict == VINF_SUCCESS
9449	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9450
9451	if (rcStrict == VINF_SUCCESS)
9452	{
9453	if (iGReg == X86_GREG_xSP)
9454	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
9455	\| HM_CHANGED_GUEST_RSP);
9456	else
9457	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9458	}
9459	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9460	{
9461	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9462	rcStrict = VINF_SUCCESS;
9463	}
9464
9465	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
9466	}
9467
9468	return rcStrict;
9469	}
9470
9471
9472	/**
9473	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9474	* Conditional VM-exit.
9475	*/
9476	HMVMX_EXIT_DECL vmxHCExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9477	{
9478	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9479
9480	#ifndef IN_NEM_DARWIN
9481	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
9482
9483	vmxHCReadToTransient< HMVMX_READ_EXIT_INSTR_LEN
9484	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
9485	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
9486	\| HMVMX_READ_IDT_VECTORING_INFO
9487	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
9488	\| HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
9489
9490	/*
9491	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
9492	*/
9493	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
9494	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9495	{
9496	/*
9497	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
9498	* instruction emulation to inject the original event. Otherwise, injecting the original event
9499	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
9500	*/
9501	if (!VCPU_2_VMXSTATE(pVCpu).Event.fPending)
9502	{ /* likely */ }
9503	else
9504	{
9505	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
9506	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9507	/** @todo NSTVMX: Think about how this should be handled. */
9508	if (pVmxTransient->fIsNestedGuest)
9509	return VERR_VMX_IPE_3;
9510	# endif
9511	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9512	}
9513	}
9514	else
9515	{
9516	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
9517	return rcStrict;
9518	}
9519
9520	/*
9521	* Get sufficient state and update the exit history entry.
9522	*/
9523	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9524	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
9525	AssertRCReturn(rc, rc);
9526
9527	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
9528	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
9529	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
9530	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
9531	if (!pExitRec)
9532	{
9533	/*
9534	* If we succeed, resume guest execution.
9535	* If we fail in interpreting the instruction because we couldn't get the guest physical address
9536	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
9537	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
9538	* weird case. See @bugref{6043}.
9539	*/
9540	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9541	/** @todo bird: We can probably just go straight to IOM here and assume that
9542	* it's MMIO, then fall back on PGM if that hunch didn't work out so
9543	* well. However, we need to address that aliasing workarounds that
9544	* PGMR0Trap0eHandlerNPMisconfig implements. So, some care is needed.
9545	*
9546	* Might also be interesting to see if we can get this done more or
9547	* less locklessly inside IOM. Need to consider the lookup table
9548	* updating and use a bit more carefully first (or do all updates via
9549	* rendezvous) */
9550	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, &pVCpu->cpum.GstCtx, GCPhys, UINT32_MAX);
9551	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
9552	if ( rcStrict == VINF_SUCCESS
9553	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
9554	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
9555	{
9556	/* Successfully handled MMIO operation. */
9557	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9558	\| HM_CHANGED_GUEST_APIC_TPR);
9559	rcStrict = VINF_SUCCESS;
9560	}
9561	}
9562	else
9563	{
9564	/*
9565	* Frequent exit or something needing probing. Call EMHistoryExec.
9566	*/
9567	int rc2 = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL, IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
9568	AssertRCReturn(rc2, rc2);
9569	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
9570	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
9571
9572	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
9573	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9574
9575	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
9576	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9577	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
9578	}
9579	return rcStrict;
9580	#else
9581	AssertFailed();
9582	return VERR_VMX_IPE_3; /* Should never happen with Apple HV in R3. */
9583	#endif
9584	}
9585
9586
9587	/**
9588	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
9589	* VM-exit.
9590	*/
9591	HMVMX_EXIT_DECL vmxHCExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9592	{
9593	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9594	#ifndef IN_NEM_DARWIN
9595	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
9596
9597	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9598	\| HMVMX_READ_EXIT_INSTR_LEN
9599	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
9600	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
9601	\| HMVMX_READ_IDT_VECTORING_INFO
9602	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
9603	\| HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
9604
9605	/*
9606	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
9607	*/
9608	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
9609	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9610	{
9611	/*
9612	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
9613	* we shall resolve the nested #PF and re-inject the original event.
9614	*/
9615	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
9616	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflectNPF);
9617	}
9618	else
9619	{
9620	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
9621	return rcStrict;
9622	}
9623
9624	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9625	int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
9626	AssertRCReturn(rc, rc);
9627
9628	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
9629	uint64_t const uExitQual = pVmxTransient->uExitQual;
9630	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
9631
9632	RTGCUINT uErrorCode = 0;
9633	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH)
9634	uErrorCode \|= X86_TRAP_PF_ID;
9635	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
9636	uErrorCode \|= X86_TRAP_PF_RW;
9637	if (uExitQual & (VMX_EXIT_QUAL_EPT_ENTRY_READ \| VMX_EXIT_QUAL_EPT_ENTRY_WRITE \| VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE))
9638	uErrorCode \|= X86_TRAP_PF_P;
9639
9640	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9641	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%08RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
9642
9643	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9644
9645	/*
9646	* Handle the pagefault trap for the nested shadow table.
9647	*/
9648	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
9649	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, pCtx, GCPhys);
9650	TRPMResetTrap(pVCpu);
9651
9652	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
9653	if ( rcStrict == VINF_SUCCESS
9654	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
9655	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
9656	{
9657	/* Successfully synced our nested page tables. */
9658	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitReasonNpf);
9659	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
9660	return VINF_SUCCESS;
9661	}
9662	Log4Func(("EPT return to ring-3 rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9663	return rcStrict;
9664
9665	#else /* IN_NEM_DARWIN */
9666	PVM pVM = pVCpu->CTX_SUFF(pVM);
9667	uint64_t const uHostTsc = ASMReadTSC(); RT_NOREF(uHostTsc);
9668	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9669	\| HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
9670	vmxHCImportGuestRip(pVCpu);
9671	vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
9672
9673	/*
9674	* Ask PGM for information about the given GCPhys. We need to check if we're
9675	* out of sync first.
9676	*/
9677	NEMHCDARWINHMACPCCSTATE State = { RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE),
9678	false,
9679	false };
9680	PGMPHYSNEMPAGEINFO Info;
9681	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pVmxTransient->uGuestPhysicalAddr, State.fWriteAccess, &Info,
9682	nemR3DarwinHandleMemoryAccessPageCheckerCallback, &State);
9683	if (RT_SUCCESS(rc))
9684	{
9685	if (Info.fNemProt & ( RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
9686	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
9687	{
9688	if (State.fCanResume)
9689	{
9690	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting\n",
9691	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9692	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
9693	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
9694	State.fDidSomething ? "" : " no-change"));
9695	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
9696	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
9697	return VINF_SUCCESS;
9698	}
9699	}
9700
9701	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating\n",
9702	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9703	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
9704	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
9705	State.fDidSomething ? "" : " no-change"));
9706	}
9707	else
9708	Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating\n",
9709	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9710	pVmxTransient->uGuestPhysicalAddr, rc, State.fDidSomething ? " modified-backing" : ""));
9711
9712	/*
9713	* Emulate the memory access, either access handler or special memory.
9714	*/
9715	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
9716	RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
9717	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
9718	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
9719	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
9720
9721	rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
9722	AssertRCReturn(rc, rc);
9723
9724	VBOXSTRICTRC rcStrict;
9725	if (!pExitRec)
9726	rcStrict = IEMExecOne(pVCpu);
9727	else
9728	{
9729	/* Frequent access or probing. */
9730	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
9731	Log4(("MemExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
9732	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9733	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
9734	}
9735
9736	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9737
9738	Log4Func(("EPT return rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9739	return rcStrict;
9740	#endif /* IN_NEM_DARWIN */
9741	}
9742
9743	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9744
9745	/**
9746	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
9747	*/
9748	HMVMX_EXIT_DECL vmxHCExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9749	{
9750	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9751
9752	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9753	\| HMVMX_READ_EXIT_INSTR_INFO
9754	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9755	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP
9756	\| CPUMCTX_EXTRN_SREG_MASK
9757	\| CPUMCTX_EXTRN_HWVIRT
9758	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
9759	AssertRCReturn(rc, rc);
9760
9761	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9762
9763	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9764	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9765
9766	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
9767	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9768	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9769	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9770	{
9771	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9772	rcStrict = VINF_SUCCESS;
9773	}
9774	return rcStrict;
9775	}
9776
9777
9778	/**
9779	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
9780	*/
9781	HMVMX_EXIT_DECL vmxHCExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9782	{
9783	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9784
9785	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
9786	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
9787	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9788	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
9789	AssertRCReturn(rc, rc);
9790
9791	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9792
9793	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9794	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
9795	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9796	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9797	{
9798	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9799	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9800	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
9801	}
9802	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
9803	return rcStrict;
9804	}
9805
9806
9807	/**
9808	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
9809	*/
9810	HMVMX_EXIT_DECL vmxHCExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9811	{
9812	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9813
9814	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9815	\| HMVMX_READ_EXIT_INSTR_INFO
9816	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9817	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP
9818	\| CPUMCTX_EXTRN_SREG_MASK
9819	\| CPUMCTX_EXTRN_HWVIRT
9820	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
9821	AssertRCReturn(rc, rc);
9822
9823	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9824
9825	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9826	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9827
9828	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
9829	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9830	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9831	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9832	{
9833	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9834	rcStrict = VINF_SUCCESS;
9835	}
9836	return rcStrict;
9837	}
9838
9839
9840	/**
9841	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
9842	*/
9843	HMVMX_EXIT_DECL vmxHCExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9844	{
9845	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9846
9847	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9848	\| HMVMX_READ_EXIT_INSTR_INFO
9849	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9850	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP
9851	\| CPUMCTX_EXTRN_SREG_MASK
9852	\| CPUMCTX_EXTRN_HWVIRT
9853	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
9854	AssertRCReturn(rc, rc);
9855
9856	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9857
9858	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9859	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
9860
9861	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
9862	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9863	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9864	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9865	{
9866	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9867	rcStrict = VINF_SUCCESS;
9868	}
9869	return rcStrict;
9870	}
9871
9872
9873	/**
9874	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
9875	*/
9876	HMVMX_EXIT_DECL vmxHCExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9877	{
9878	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9879
9880	/*
9881	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
9882	* thus might not need to import the shadow VMCS state, it's safer just in case
9883	* code elsewhere dares look at unsynced VMCS fields.
9884	*/
9885	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9886	\| HMVMX_READ_EXIT_INSTR_INFO
9887	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9888	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP
9889	\| CPUMCTX_EXTRN_SREG_MASK
9890	\| CPUMCTX_EXTRN_HWVIRT
9891	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
9892	AssertRCReturn(rc, rc);
9893
9894	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9895
9896	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9897	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
9898	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
9899
9900	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
9901	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9902	{
9903	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9904
9905	# if 0 //ndef IN_NEM_DARWIN /** @todo this needs serious tuning still, slows down things enormously. */
9906	/* Try for exit optimization. This is on the following instruction
9907	because it would be a waste of time to have to reinterpret the
9908	already decoded vmwrite instruction. */
9909	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndType(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_VMREAD));
9910	if (pExitRec)
9911	{
9912	/* Frequent access or probing. */
9913	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9914	AssertRCReturn(rc, rc);
9915
9916	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
9917	Log4(("vmread/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
9918	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9919	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
9920	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9921	}
9922	# endif
9923	}
9924	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9925	{
9926	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9927	rcStrict = VINF_SUCCESS;
9928	}
9929	return rcStrict;
9930	}
9931
9932
9933	/**
9934	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
9935	*/
9936	HMVMX_EXIT_DECL vmxHCExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9937	{
9938	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9939
9940	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
9941	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
9942	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9943	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
9944	AssertRCReturn(rc, rc);
9945
9946	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9947
9948	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9949	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
9950	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9951	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9952	{
9953	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9954	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9955	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
9956	}
9957	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
9958	return rcStrict;
9959	}
9960
9961
9962	/**
9963	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
9964	*/
9965	HMVMX_EXIT_DECL vmxHCExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9966	{
9967	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9968
9969	/*
9970	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
9971	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
9972	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
9973	*/
9974	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
9975	\| HMVMX_READ_EXIT_INSTR_INFO
9976	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
9977	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP
9978	\| CPUMCTX_EXTRN_SREG_MASK
9979	\| CPUMCTX_EXTRN_HWVIRT
9980	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
9981	AssertRCReturn(rc, rc);
9982
9983	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9984
9985	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
9986	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
9987	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9988
9989	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
9990	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9991	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9992	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9993	{
9994	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9995	rcStrict = VINF_SUCCESS;
9996	}
9997	return rcStrict;
9998	}
9999
10000
10001	/**
10002	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
10003	*/
10004	HMVMX_EXIT_DECL vmxHCExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10005	{
10006	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10007
10008	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10009	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_CR4
10010	\| CPUMCTX_EXTRN_HWVIRT
10011	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
10012	AssertRCReturn(rc, rc);
10013
10014	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
10015
10016	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
10017	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10018	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
10019	else if (rcStrict == VINF_IEM_RAISED_XCPT)
10020	{
10021	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10022	rcStrict = VINF_SUCCESS;
10023	}
10024	return rcStrict;
10025	}
10026
10027
10028	/**
10029	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
10030	*/
10031	HMVMX_EXIT_DECL vmxHCExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10032	{
10033	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10034
10035	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10036	\| HMVMX_READ_EXIT_INSTR_INFO
10037	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10038	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP
10039	\| CPUMCTX_EXTRN_SREG_MASK
10040	\| CPUMCTX_EXTRN_HWVIRT
10041	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
10042	AssertRCReturn(rc, rc);
10043
10044	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
10045
10046	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
10047	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
10048
10049	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
10050	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10051	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
10052	else if (rcStrict == VINF_IEM_RAISED_XCPT)
10053	{
10054	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10055	rcStrict = VINF_SUCCESS;
10056	}
10057	return rcStrict;
10058	}
10059
10060
10061	/**
10062	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
10063	*/
10064	HMVMX_EXIT_DECL vmxHCExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10065	{
10066	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10067
10068	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10069	\| HMVMX_READ_EXIT_INSTR_INFO
10070	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10071	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP
10072	\| CPUMCTX_EXTRN_SREG_MASK
10073	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
10074	AssertRCReturn(rc, rc);
10075
10076	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
10077
10078	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
10079	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
10080
10081	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
10082	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10083	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10084	else if (rcStrict == VINF_IEM_RAISED_XCPT)
10085	{
10086	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10087	rcStrict = VINF_SUCCESS;
10088	}
10089	return rcStrict;
10090	}
10091
10092
10093	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
10094	/**
10095	* VM-exit handler for INVEPT (VMX_EXIT_INVEPT). Unconditional VM-exit.
10096	*/
10097	HMVMX_EXIT_DECL vmxHCExitInvept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10098	{
10099	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10100
10101	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10102	\| HMVMX_READ_EXIT_INSTR_INFO
10103	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10104	int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP
10105	\| CPUMCTX_EXTRN_SREG_MASK
10106	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
10107	AssertRCReturn(rc, rc);
10108
10109	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
10110
10111	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
10112	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
10113
10114	VBOXSTRICTRC rcStrict = IEMExecDecodedInvept(pVCpu, &ExitInfo);
10115	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10116	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10117	else if (rcStrict == VINF_IEM_RAISED_XCPT)
10118	{
10119	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10120	rcStrict = VINF_SUCCESS;
10121	}
10122	return rcStrict;
10123	}
10124	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
10125	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10126	/** @} */
10127
10128
10129	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10130	/** @name Nested-guest VM-exit handlers.
10131	* @{
10132	*/
10133	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10134	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10135	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10136
10137	/**
10138	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10139	* Conditional VM-exit.
10140	*/
10141	HMVMX_EXIT_DECL vmxHCExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10142	{
10143	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10144
10145	vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
10146
10147	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
10148	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
10149	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
10150
10151	switch (uExitIntType)
10152	{
10153	# ifndef IN_NEM_DARWIN
10154	/*
10155	* Physical NMIs:
10156	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
10157	*/
10158	case VMX_EXIT_INT_INFO_TYPE_NMI:
10159	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
10160	# endif
10161
10162	/*
10163	* Hardware exceptions,
10164	* Software exceptions,
10165	* Privileged software exceptions:
10166	* Figure out if the exception must be delivered to the guest or the nested-guest.
10167	*/
10168	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
10169	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
10170	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
10171	{
10172	vmxHCReadToTransient< HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
10173	\| HMVMX_READ_EXIT_INSTR_LEN
10174	\| HMVMX_READ_IDT_VECTORING_INFO
10175	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
10176
10177	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10178	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
10179	if (CPUMIsGuestVmxXcptInterceptSet(pCtx, uVector, pVmxTransient->uExitIntErrorCode))
10180	{
10181	/*
10182	* Split-lock triggered #ACs should not be injected into the nested-guest
10183	* since we don't support split-lock detection for nested-guests yet.
10184	*/
10185	if ( uVector == X86_XCPT_AC
10186	&& uExitIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
10187	{
10188	int const rc = vmxHCImportGuestState<HMVMX_CPUMCTX_XPCT_AC>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
10189	AssertRCReturn(rc, rc);
10190	if (vmxHCIsSplitLockAcXcpt(pVCpu))
10191	{
10192	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10193	if ( rcStrict == VINF_SUCCESS
10194	&& !VCPU_2_VMXSTATE(pVCpu).Event.fPending)
10195	return vmxHCHandleSplitLockAcXcpt(pVCpu, pVmxTransient);
10196	if (rcStrict == VINF_HM_DOUBLE_FAULT)
10197	{
10198	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
10199	rcStrict = VINF_SUCCESS;
10200	}
10201	return rcStrict;
10202	}
10203	}
10204
10205	/* Exit qualification is required for debug and page-fault exceptions. */
10206	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
10207
10208	/*
10209	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
10210	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
10211	* length. However, if delivery of a software interrupt, software exception or privileged
10212	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
10213	*/
10214	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
10215	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT(pVmxTransient->uExitIntInfo,
10216	pVmxTransient->uExitIntErrorCode,
10217	pVmxTransient->uIdtVectoringInfo,
10218	pVmxTransient->uIdtVectoringErrorCode);
10219	#ifdef DEBUG_ramshankar
10220	vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10221	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n",
10222	pVmxTransient->uExitIntInfo, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
10223	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
10224	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n",
10225	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
10226	#endif
10227	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
10228	}
10229
10230	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in vmxHCExitXcptPF. */
10231	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
10232	return vmxHCExitXcpt(pVCpu, pVmxTransient);
10233	}
10234
10235	/*
10236	* Software interrupts:
10237	* VM-exits cannot be caused by software interrupts.
10238	*
10239	* External interrupts:
10240	* This should only happen when "acknowledge external interrupts on VM-exit"
10241	* control is set. However, we never set this when executing a guest or
10242	* nested-guest. For nested-guests it is emulated while injecting interrupts into
10243	* the guest.
10244	*/
10245	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
10246	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
10247	default:
10248	{
10249	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
10250	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10251	}
10252	}
10253	}
10254
10255
10256	/**
10257	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
10258	* Unconditional VM-exit.
10259	*/
10260	HMVMX_EXIT_DECL vmxHCExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10261	{
10262	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10263	return IEMExecVmxVmexitTripleFault(pVCpu);
10264	}
10265
10266
10267	/**
10268	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10269	*/
10270	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10271	{
10272	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10273
10274	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
10275	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
10276	return vmxHCExitIntWindow(pVCpu, pVmxTransient);
10277	}
10278
10279
10280	/**
10281	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10282	*/
10283	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10284	{
10285	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10286
10287	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
10288	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
10289	return vmxHCExitNmiWindow(pVCpu, pVmxTransient);
10290	}
10291
10292
10293	/**
10294	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
10295	* Unconditional VM-exit.
10296	*/
10297	HMVMX_EXIT_DECL vmxHCExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10298	{
10299	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10300
10301	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10302	\| HMVMX_READ_EXIT_INSTR_LEN
10303	\| HMVMX_READ_IDT_VECTORING_INFO
10304	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
10305
10306	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
10307	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
10308	pVmxTransient->uIdtVectoringErrorCode);
10309	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
10310	}
10311
10312
10313	/**
10314	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10315	*/
10316	HMVMX_EXIT_DECL vmxHCExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10317	{
10318	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10319
10320	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
10321	{
10322	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10323	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10324	}
10325	return vmxHCExitHlt(pVCpu, pVmxTransient);
10326	}
10327
10328
10329	/**
10330	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10331	*/
10332	HMVMX_EXIT_DECL vmxHCExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10333	{
10334	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10335
10336	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
10337	{
10338	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10339	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10340	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
10341	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10342	}
10343	return vmxHCExitInvlpg(pVCpu, pVmxTransient);
10344	}
10345
10346
10347	/**
10348	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10349	*/
10350	HMVMX_EXIT_DECL vmxHCExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10351	{
10352	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10353
10354	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
10355	{
10356	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10357	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10358	}
10359	return vmxHCExitRdpmc(pVCpu, pVmxTransient);
10360	}
10361
10362
10363	/**
10364	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
10365	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
10366	*/
10367	HMVMX_EXIT_DECL vmxHCExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10368	{
10369	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10370
10371	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
10372	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
10373
10374	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
10375
10376	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
10377	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
10378	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
10379
10380	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
10381	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10382	u64VmcsField &= UINT64_C(0xffffffff);
10383
10384	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
10385	{
10386	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10387	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10388	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
10389	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10390	}
10391
10392	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
10393	return vmxHCExitVmread(pVCpu, pVmxTransient);
10394	return vmxHCExitVmwrite(pVCpu, pVmxTransient);
10395	}
10396
10397
10398	/**
10399	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10400	*/
10401	HMVMX_EXIT_DECL vmxHCExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10402	{
10403	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10404
10405	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
10406	{
10407	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10408	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10409	}
10410
10411	return vmxHCExitRdtsc(pVCpu, pVmxTransient);
10412	}
10413
10414
10415	/**
10416	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
10417	* Conditional VM-exit.
10418	*/
10419	HMVMX_EXIT_DECL vmxHCExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10420	{
10421	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10422
10423	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10424	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10425
10426	VBOXSTRICTRC rcStrict;
10427	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
10428	switch (uAccessType)
10429	{
10430	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
10431	{
10432	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
10433	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
10434	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
10435	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
10436
10437	bool fIntercept;
10438	switch (iCrReg)
10439	{
10440	case 0:
10441	case 4:
10442	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
10443	break;
10444
10445	case 3:
10446	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
10447	break;
10448
10449	case 8:
10450	fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
10451	break;
10452
10453	default:
10454	fIntercept = false;
10455	break;
10456	}
10457	if (fIntercept)
10458	{
10459	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
10460	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10461	}
10462	else
10463	{
10464	int const rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
10465	AssertRCReturn(rc, rc);
10466	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
10467	}
10468	break;
10469	}
10470
10471	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
10472	{
10473	/*
10474	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
10475	* CR2 reads do not cause a VM-exit.
10476	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
10477	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
10478	*/
10479	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
10480	if ( iCrReg == 3
10481	\|\| iCrReg == 8)
10482	{
10483	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
10484	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
10485	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
10486	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
10487	{
10488	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
10489	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10490	}
10491	else
10492	{
10493	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
10494	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
10495	}
10496	}
10497	else
10498	{
10499	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
10500	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
10501	}
10502	break;
10503	}
10504
10505	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
10506	{
10507	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
10508	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
10509	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
10510	if ( (uGstHostMask & X86_CR0_TS)
10511	&& (uReadShadow & X86_CR0_TS))
10512	{
10513	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
10514	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10515	}
10516	else
10517	rcStrict = vmxHCExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
10518	break;
10519	}
10520
10521	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10522	{
10523	RTGCPTR GCPtrEffDst;
10524	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
10525	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
10526	if (fMemOperand)
10527	{
10528	vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
10529	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
10530	}
10531	else
10532	GCPtrEffDst = NIL_RTGCPTR;
10533
10534	if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
10535	{
10536	VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
10537	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
10538	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10539	}
10540	else
10541	rcStrict = vmxHCExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
10542	break;
10543	}
10544
10545	default:
10546	{
10547	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
10548	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
10549	}
10550	}
10551
10552	if (rcStrict == VINF_IEM_RAISED_XCPT)
10553	{
10554	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
10555	rcStrict = VINF_SUCCESS;
10556	}
10557	return rcStrict;
10558	}
10559
10560
10561	/**
10562	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
10563	* Conditional VM-exit.
10564	*/
10565	HMVMX_EXIT_DECL vmxHCExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10566	{
10567	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10568
10569	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
10570	{
10571	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10572	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10573	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
10574	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10575	}
10576	return vmxHCExitMovDRx(pVCpu, pVmxTransient);
10577	}
10578
10579
10580	/**
10581	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
10582	* Conditional VM-exit.
10583	*/
10584	HMVMX_EXIT_DECL vmxHCExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10585	{
10586	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10587
10588	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
10589
10590	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
10591	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
10592	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
10593
10594	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
10595	uint8_t const cbAccess = s_aIOSizes[uIOSize];
10596	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
10597	{
10598	/*
10599	* IN/OUT instruction:
10600	* - Provides VM-exit instruction length.
10601	*
10602	* INS/OUTS instruction:
10603	* - Provides VM-exit instruction length.
10604	* - Provides Guest-linear address.
10605	* - Optionally provides VM-exit instruction info (depends on CPU feature).
10606	*/
10607	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10608	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10609
10610	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
10611	pVmxTransient->ExitInstrInfo.u = 0;
10612	pVmxTransient->uGuestLinearAddr = 0;
10613
10614	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
10615	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
10616	if (fIOString)
10617	{
10618	vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
10619	if (fVmxInsOutsInfo)
10620	{
10621	Assert(RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
10622	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
10623	}
10624	}
10625
10626	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_AND_LIN_ADDR_FROM_TRANSIENT(pVmxTransient);
10627	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10628	}
10629	return vmxHCExitIoInstr(pVCpu, pVmxTransient);
10630	}
10631
10632
10633	/**
10634	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10635	*/
10636	HMVMX_EXIT_DECL vmxHCExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10637	{
10638	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10639
10640	uint32_t fMsrpm;
10641	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
10642	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
10643	else
10644	fMsrpm = VMXMSRPM_EXIT_RD;
10645
10646	if (fMsrpm & VMXMSRPM_EXIT_RD)
10647	{
10648	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10649	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10650	}
10651	return vmxHCExitRdmsr(pVCpu, pVmxTransient);
10652	}
10653
10654
10655	/**
10656	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10657	*/
10658	HMVMX_EXIT_DECL vmxHCExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10659	{
10660	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10661
10662	uint32_t fMsrpm;
10663	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
10664	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
10665	else
10666	fMsrpm = VMXMSRPM_EXIT_WR;
10667
10668	if (fMsrpm & VMXMSRPM_EXIT_WR)
10669	{
10670	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10671	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10672	}
10673	return vmxHCExitWrmsr(pVCpu, pVmxTransient);
10674	}
10675
10676
10677	/**
10678	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10679	*/
10680	HMVMX_EXIT_DECL vmxHCExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10681	{
10682	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10683
10684	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
10685	{
10686	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10687	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10688	}
10689	return vmxHCExitMwait(pVCpu, pVmxTransient);
10690	}
10691
10692
10693	/**
10694	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
10695	* VM-exit.
10696	*/
10697	HMVMX_EXIT_DECL vmxHCExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10698	{
10699	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10700
10701	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
10702	vmxHCReadToTransient<HMVMX_READ_GUEST_PENDING_DBG_XCPTS>(pVCpu, pVmxTransient);
10703	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient);
10704	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
10705	}
10706
10707
10708	/**
10709	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10710	*/
10711	HMVMX_EXIT_DECL vmxHCExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10712	{
10713	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10714
10715	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
10716	{
10717	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10718	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10719	}
10720	return vmxHCExitMonitor(pVCpu, pVmxTransient);
10721	}
10722
10723
10724	/**
10725	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10726	*/
10727	HMVMX_EXIT_DECL vmxHCExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10728	{
10729	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10730
10731	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
10732	* PAUSE when executing a nested-guest? If it does not, we would not need
10733	* to check for the intercepts here. Just call VM-exit... */
10734
10735	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
10736	if ( CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
10737	\|\| CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10738	{
10739	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10740	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10741	}
10742	return vmxHCExitPause(pVCpu, pVmxTransient);
10743	}
10744
10745
10746	/**
10747	* Nested-guest VM-exit handler for when the TPR value is lowered below the
10748	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10749	*/
10750	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10751	{
10752	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10753
10754	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
10755	{
10756	vmxHCReadToTransient<HMVMX_READ_GUEST_PENDING_DBG_XCPTS>(pVCpu, pVmxTransient);
10757	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient);
10758	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
10759	}
10760	return vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient);
10761	}
10762
10763
10764	/**
10765	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
10766	* VM-exit.
10767	*/
10768	HMVMX_EXIT_DECL vmxHCExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10769	{
10770	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10771
10772	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10773	\| HMVMX_READ_EXIT_INSTR_LEN
10774	\| HMVMX_READ_IDT_VECTORING_INFO
10775	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
10776
10777	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
10778
10779	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
10780	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
10781
10782	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
10783	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
10784	pVmxTransient->uIdtVectoringErrorCode);
10785	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
10786	}
10787
10788
10789	/**
10790	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
10791	* Conditional VM-exit.
10792	*/
10793	HMVMX_EXIT_DECL vmxHCExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10794	{
10795	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10796
10797	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
10798	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
10799	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
10800	}
10801
10802
10803	/**
10804	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
10805	* Conditional VM-exit.
10806	*/
10807	HMVMX_EXIT_DECL vmxHCExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10808	{
10809	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10810
10811	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
10812	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
10813	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
10814	}
10815
10816
10817	/**
10818	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10819	*/
10820	HMVMX_EXIT_DECL vmxHCExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10821	{
10822	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10823
10824	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
10825	{
10826	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
10827	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10828	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10829	}
10830	return vmxHCExitRdtscp(pVCpu, pVmxTransient);
10831	}
10832
10833
10834	/**
10835	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10836	*/
10837	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10838	{
10839	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10840
10841	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
10842	{
10843	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10844	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10845	}
10846	return vmxHCExitWbinvd(pVCpu, pVmxTransient);
10847	}
10848
10849
10850	/**
10851	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10852	*/
10853	HMVMX_EXIT_DECL vmxHCExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10854	{
10855	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10856
10857	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
10858	{
10859	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
10860	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10861	\| HMVMX_READ_EXIT_INSTR_INFO
10862	\| HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10863	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
10864	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10865	}
10866	return vmxHCExitInvpcid(pVCpu, pVmxTransient);
10867	}
10868
10869
10870	/**
10871	* Nested-guest VM-exit handler for invalid-guest state
10872	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
10873	*/
10874	HMVMX_EXIT_DECL vmxHCExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10875	{
10876	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10877
10878	/*
10879	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
10880	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
10881	* Handle it like it's in an invalid guest state of the outer guest.
10882	*
10883	* When the fast path is implemented, this should be changed to cause the corresponding
10884	* nested-guest VM-exit.
10885	*/
10886	return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
10887	}
10888
10889
10890	/**
10891	* Nested-guest VM-exit handler for instructions that cause VM-exits unconditionally
10892	* and only provide the instruction length.
10893	*
10894	* Unconditional VM-exit.
10895	*/
10896	HMVMX_EXIT_DECL vmxHCExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10897	{
10898	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10899
10900	#ifdef VBOX_STRICT
10901	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10902	switch (pVmxTransient->uExitReason)
10903	{
10904	case VMX_EXIT_ENCLS:
10905	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
10906	break;
10907
10908	case VMX_EXIT_VMFUNC:
10909	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
10910	break;
10911	}
10912	#endif
10913
10914	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
10915	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10916	}
10917
10918
10919	/**
10920	* Nested-guest VM-exit handler for instructions that provide instruction length as
10921	* well as more information.
10922	*
10923	* Unconditional VM-exit.
10924	*/
10925	HMVMX_EXIT_DECL vmxHCExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10926	{
10927	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10928
10929	# ifdef VBOX_STRICT
10930	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10931	switch (pVmxTransient->uExitReason)
10932	{
10933	case VMX_EXIT_GDTR_IDTR_ACCESS:
10934	case VMX_EXIT_LDTR_TR_ACCESS:
10935	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
10936	break;
10937
10938	case VMX_EXIT_RDRAND:
10939	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
10940	break;
10941
10942	case VMX_EXIT_RDSEED:
10943	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
10944	break;
10945
10946	case VMX_EXIT_XSAVES:
10947	case VMX_EXIT_XRSTORS:
10948	/** @todo NSTVMX: Verify XSS-bitmap. */
10949	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
10950	break;
10951
10952	case VMX_EXIT_UMWAIT:
10953	case VMX_EXIT_TPAUSE:
10954	Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
10955	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
10956	break;
10957
10958	case VMX_EXIT_LOADIWKEY:
10959	Assert(CPUMIsGuestVmxProcCtls3Set(pCtx, VMX_PROC_CTLS3_LOADIWKEY_EXIT));
10960	break;
10961	}
10962	# endif
10963
10964	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10965	\| HMVMX_READ_EXIT_INSTR_LEN
10966	\| HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
10967	VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
10968	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10969	}
10970
10971	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
10972
10973	/**
10974	* Nested-guest VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION).
10975	* Conditional VM-exit.
10976	*/
10977	HMVMX_EXIT_DECL vmxHCExitEptViolationNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10978	{
10979	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10980	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
10981
10982	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10983	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT))
10984	{
10985	vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION
10986	\| HMVMX_READ_EXIT_INSTR_LEN
10987	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
10988	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
10989	\| HMVMX_READ_IDT_VECTORING_INFO
10990	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
10991	\| HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
10992	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
10993	AssertRCReturn(rc, rc);
10994
10995	/*
10996	* If it's our VMEXIT, we're responsible for re-injecting any event which delivery
10997	* might have triggered this VMEXIT. If we forward the problem to the inner VMM,
10998	* it's its problem to deal with that issue and we'll clear the recovered event.
10999	*/
11000	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11001	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11002	{ /likely/ }
11003	else
11004	{
11005	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
11006	return rcStrict;
11007	}
11008	uint32_t const fClearEventOnForward = VCPU_2_VMXSTATE(pVCpu).Event.fPending; /* paranoia. should not inject events below. */
11009
11010	RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr;
11011	uint64_t const uExitQual = pVmxTransient->uExitQual;
11012
11013	RTGCPTR GCPtrNestedFault;
11014	bool const fIsLinearAddrValid = RT_BOOL(uExitQual & VMX_EXIT_QUAL_EPT_LINEAR_ADDR_VALID);
11015	if (fIsLinearAddrValid)
11016	{
11017	vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
11018	GCPtrNestedFault = pVmxTransient->uGuestLinearAddr;
11019	}
11020	else
11021	GCPtrNestedFault = 0;
11022
11023	RTGCUINT const uErr = ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH) ? X86_TRAP_PF_ID : 0)
11024	\| ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE) ? X86_TRAP_PF_RW : 0)
11025	\| ((uExitQual & ( VMX_EXIT_QUAL_EPT_ENTRY_READ
11026	\| VMX_EXIT_QUAL_EPT_ENTRY_WRITE
11027	\| VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE)) ? X86_TRAP_PF_P : 0);
11028
11029	PGMPTWALK Walk;
11030	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11031	rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, uErr, pCtx, GCPhysNestedFault,
11032	fIsLinearAddrValid, GCPtrNestedFault, &Walk);
11033	Log7Func(("PGM (uExitQual=%#RX64, %RGp, %RGv) -> %Rrc (fFailed=%d)\n",
11034	uExitQual, GCPhysNestedFault, GCPtrNestedFault, VBOXSTRICTRC_VAL(rcStrict), Walk.fFailed));
11035	if (RT_SUCCESS(rcStrict))
11036	{
11037	if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
11038	{
11039	Assert(!fClearEventOnForward);
11040	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IOM));
11041	rcStrict = VINF_EM_RESCHEDULE_REM;
11042	}
11043	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
11044	return rcStrict;
11045	}
11046
11047	if (fClearEventOnForward)
11048	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
11049
11050	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
11051	pVmxTransient->uIdtVectoringErrorCode);
11052	if (Walk.fFailed & PGM_WALKFAIL_EPT_VIOLATION)
11053	{
11054	VMXVEXITINFO const ExitInfo
11055	= VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_AND_GST_ADDRESSES(VMX_EXIT_EPT_VIOLATION,
11056	pVmxTransient->uExitQual,
11057	pVmxTransient->cbExitInstr,
11058	pVmxTransient->uGuestLinearAddr,
11059	pVmxTransient->uGuestPhysicalAddr);
11060	return IEMExecVmxVmexitEptViolation(pVCpu, &ExitInfo, &ExitEventInfo);
11061	}
11062
11063	AssertMsgReturn(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG,
11064	("uErr=%#RX32 uExitQual=%#RX64 GCPhysNestedFault=%#RGp GCPtrNestedFault=%#RGv\n",
11065	(uint32_t)uErr, uExitQual, GCPhysNestedFault, GCPtrNestedFault),
11066	rcStrict);
11067	return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo);
11068	}
11069
11070	return vmxHCExitEptViolation(pVCpu, pVmxTransient);
11071	}
11072
11073
11074	/**
11075	* Nested-guest VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11076	* Conditional VM-exit.
11077	*/
11078	HMVMX_EXIT_DECL vmxHCExitEptMisconfigNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
11079	{
11080	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11081	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
11082
11083	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11084	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT))
11085	{
11086	vmxHCReadToTransient<HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
11087	int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
11088	AssertRCReturn(rc, rc);
11089
11090	PGMPTWALK Walk;
11091	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11092	RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr;
11093	VBOXSTRICTRC rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, X86_TRAP_PF_RSVD, pCtx,
11094	GCPhysNestedFault, false /* fIsLinearAddrValid */,
11095	0 /* GCPtrNestedFault */, &Walk);
11096	if (RT_SUCCESS(rcStrict))
11097	{
11098	AssertMsgFailed(("Shouldn't happen with the way we have programmed the EPT shadow tables\n"));
11099	return rcStrict;
11100	}
11101
11102	AssertMsg(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG, ("GCPhysNestedFault=%#RGp\n", GCPhysNestedFault));
11103	vmxHCReadToTransient< HMVMX_READ_IDT_VECTORING_INFO
11104	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
11105
11106	VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
11107	pVmxTransient->uIdtVectoringErrorCode);
11108	return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo);
11109	}
11110
11111	return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
11112	}
11113
11114	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
11115
11116	/** @} */
11117	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11118
11119
11120	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11121	* probes.
11122	*
11123	* The following few functions and associated structure contains the bloat
11124	* necessary for providing detailed debug events and dtrace probes as well as
11125	* reliable host side single stepping. This works on the principle of
11126	* "subclassing" the normal execution loop and workers. We replace the loop
11127	* method completely and override selected helpers to add necessary adjustments
11128	* to their core operation.
11129	*
11130	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11131	* any performance for debug and analysis features.
11132	*
11133	* @{
11134	*/
11135
11136	/**
11137	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11138	* the debug run loop.
11139	*/
11140	typedef struct VMXRUNDBGSTATE
11141	{
11142	/** The RIP we started executing at. This is for detecting that we stepped. */
11143	uint64_t uRipStart;
11144	/** The CS we started executing with. */
11145	uint16_t uCsStart;
11146
11147	/** Whether we've actually modified the 1st execution control field. */
11148	bool fModifiedProcCtls : 1;
11149	/** Whether we've actually modified the 2nd execution control field. */
11150	bool fModifiedProcCtls2 : 1;
11151	/** Whether we've actually modified the exception bitmap. */
11152	bool fModifiedXcptBitmap : 1;
11153
11154	/** We desire the modified the CR0 mask to be cleared. */
11155	bool fClearCr0Mask : 1;
11156	/** We desire the modified the CR4 mask to be cleared. */
11157	bool fClearCr4Mask : 1;
11158	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11159	uint32_t fCpe1Extra;
11160	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11161	uint32_t fCpe1Unwanted;
11162	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11163	uint32_t fCpe2Extra;
11164	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11165	uint32_t bmXcptExtra;
11166	/** The sequence number of the Dtrace provider settings the state was
11167	* configured against. */
11168	uint32_t uDtraceSettingsSeqNo;
11169	/** VM-exits to check (one bit per VM-exit). */
11170	uint32_t bmExitsToCheck[3];
11171
11172	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11173	uint32_t fProcCtlsInitial;
11174	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11175	uint32_t fProcCtls2Initial;
11176	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11177	uint32_t bmXcptInitial;
11178	} VMXRUNDBGSTATE;
11179	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11180	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11181
11182
11183	/**
11184	* Initializes the VMXRUNDBGSTATE structure.
11185	*
11186	* @param pVCpu The cross context virtual CPU structure of the
11187	* calling EMT.
11188	* @param pVmxTransient The VMX-transient structure.
11189	* @param pDbgState The debug state to initialize.
11190	*/
11191	static void vmxHCRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11192	{
11193	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11194	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11195
11196	pDbgState->fModifiedProcCtls = false;
11197	pDbgState->fModifiedProcCtls2 = false;
11198	pDbgState->fModifiedXcptBitmap = false;
11199	pDbgState->fClearCr0Mask = false;
11200	pDbgState->fClearCr4Mask = false;
11201	pDbgState->fCpe1Extra = 0;
11202	pDbgState->fCpe1Unwanted = 0;
11203	pDbgState->fCpe2Extra = 0;
11204	pDbgState->bmXcptExtra = 0;
11205	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11206	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11207	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11208	}
11209
11210
11211	/**
11212	* Updates the VMSC fields with changes requested by @a pDbgState.
11213	*
11214	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11215	* immediately before executing guest code, i.e. when interrupts are disabled.
11216	* We don't check status codes here as we cannot easily assert or return in the
11217	* latter case.
11218	*
11219	* @param pVCpu The cross context virtual CPU structure.
11220	* @param pVmxTransient The VMX-transient structure.
11221	* @param pDbgState The debug state.
11222	*/
11223	static void vmxHCPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11224	{
11225	/*
11226	* Ensure desired flags in VMCS control fields are set.
11227	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11228	*
11229	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11230	* there should be no stale data in pCtx at this point.
11231	*/
11232	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11233	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11234	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11235	{
11236	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11237	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11238	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11239	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11240	pDbgState->fModifiedProcCtls = true;
11241	}
11242
11243	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11244	{
11245	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11246	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11247	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11248	pDbgState->fModifiedProcCtls2 = true;
11249	}
11250
11251	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11252	{
11253	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11254	VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11255	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11256	pDbgState->fModifiedXcptBitmap = true;
11257	}
11258
11259	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11260	{
11261	pVmcsInfo->u64Cr0Mask = 0;
11262	VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, 0);
11263	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11264	}
11265
11266	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11267	{
11268	pVmcsInfo->u64Cr4Mask = 0;
11269	VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, 0);
11270	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11271	}
11272
11273	NOREF(pVCpu);
11274	}
11275
11276
11277	/**
11278	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11279	* re-entry next time around.
11280	*
11281	* @returns Strict VBox status code (i.e. informational status codes too).
11282	* @param pVCpu The cross context virtual CPU structure.
11283	* @param pVmxTransient The VMX-transient structure.
11284	* @param pDbgState The debug state.
11285	* @param rcStrict The return code from executing the guest using single
11286	* stepping.
11287	*/
11288	static VBOXSTRICTRC vmxHCRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11289	VBOXSTRICTRC rcStrict)
11290	{
11291	/*
11292	* Restore VM-exit control settings as we may not reenter this function the
11293	* next time around.
11294	*/
11295	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11296
11297	/* We reload the initial value, trigger what we can of recalculations the
11298	next time around. From the looks of things, that's all that's required atm. */
11299	if (pDbgState->fModifiedProcCtls)
11300	{
11301	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11302	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11303	int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11304	AssertRC(rc2);
11305	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11306	}
11307
11308	/* We're currently the only ones messing with this one, so just restore the
11309	cached value and reload the field. */
11310	if ( pDbgState->fModifiedProcCtls2
11311	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11312	{
11313	int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11314	AssertRC(rc2);
11315	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11316	}
11317
11318	/* If we've modified the exception bitmap, we restore it and trigger
11319	reloading and partial recalculation the next time around. */
11320	if (pDbgState->fModifiedXcptBitmap)
11321	{
11322	int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pDbgState->bmXcptInitial);
11323	AssertRC(rc2);
11324	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11325	}
11326
11327	return rcStrict;
11328	}
11329
11330
11331	/**
11332	* Configures VM-exit controls for current DBGF and DTrace settings.
11333	*
11334	* This updates @a pDbgState and the VMCS execution control fields to reflect
11335	* the necessary VM-exits demanded by DBGF and DTrace.
11336	*
11337	* @param pVCpu The cross context virtual CPU structure.
11338	* @param pVmxTransient The VMX-transient structure. May update
11339	* fUpdatedTscOffsettingAndPreemptTimer.
11340	* @param pDbgState The debug state.
11341	*/
11342	static void vmxHCPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11343	{
11344	#ifndef IN_NEM_DARWIN
11345	/*
11346	* Take down the dtrace serial number so we can spot changes.
11347	*/
11348	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11349	ASMCompilerBarrier();
11350	#endif
11351
11352	/*
11353	* We'll rebuild most of the middle block of data members (holding the
11354	* current settings) as we go along here, so start by clearing it all.
11355	*/
11356	pDbgState->bmXcptExtra = 0;
11357	pDbgState->fCpe1Extra = 0;
11358	pDbgState->fCpe1Unwanted = 0;
11359	pDbgState->fCpe2Extra = 0;
11360	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11361	pDbgState->bmExitsToCheck[i] = 0;
11362
11363	/*
11364	* Software interrupts (INT XXh) - no idea how to trigger these...
11365	*/
11366	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11367	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11368	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11369	{
11370	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11371	}
11372
11373	/*
11374	* INT3 breakpoints - triggered by #BP exceptions.
11375	*/
11376	if (pVM->dbgf.ro.cEnabledSwBreakpoints > 0)
11377	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11378
11379	/*
11380	* Exception bitmap and XCPT events+probes.
11381	*/
11382	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11383	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11384	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11385
11386	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11387	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11388	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11389	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11390	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11391	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11392	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11393	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11394	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11395	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11396	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11397	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11398	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11399	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11400	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11401	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11402	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11403	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11404
11405	if (pDbgState->bmXcptExtra)
11406	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11407
11408	/*
11409	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11410	*
11411	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11412	* So, when adding/changing/removing please don't forget to update it.
11413	*
11414	* Some of the macros are picking up local variables to save horizontal space,
11415	* (being able to see it in a table is the lesser evil here).
11416	*/
11417	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11418	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11419	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11420	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11421	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11422	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11423	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11424	} else do { } while (0)
11425	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11426	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11427	{ \
11428	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11429	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11430	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11431	} else do { } while (0)
11432	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11433	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11434	{ \
11435	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11436	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11437	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11438	} else do { } while (0)
11439	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11440	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11441	{ \
11442	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11443	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11444	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11445	} else do { } while (0)
11446
11447	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11448	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11449	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11450	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11451	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11452
11453	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11454	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11455	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11456	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11457	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11458	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11459	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11460	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11461	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11462	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11463	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11464	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11465	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11466	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11467	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11468	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11469	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11470	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11471	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11472	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11473	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11474	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11475	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11476	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11477	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11478	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11479	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11480	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11481	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11482	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11483	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11484	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11485	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11486	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11487	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11488	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11489
11490	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11491	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11492	{
11493	int rc = vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo,
11494	CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_APIC_TPR);
11495	AssertRC(rc);
11496
11497	#if 0 /** @todo fix me */
11498	pDbgState->fClearCr0Mask = true;
11499	pDbgState->fClearCr4Mask = true;
11500	#endif
11501	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11502	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11503	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11504	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11505	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11506	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11507	require clearing here and in the loop if we start using it. */
11508	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11509	}
11510	else
11511	{
11512	if (pDbgState->fClearCr0Mask)
11513	{
11514	pDbgState->fClearCr0Mask = false;
11515	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR0);
11516	}
11517	if (pDbgState->fClearCr4Mask)
11518	{
11519	pDbgState->fClearCr4Mask = false;
11520	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR4);
11521	}
11522	}
11523	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11524	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11525
11526	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11527	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11528	{
11529	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11530	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11531	}
11532	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11533	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11534
11535	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11536	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11537	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11538	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11539	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11540	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11541	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11542	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11543	#if 0 /** @todo too slow, fix handler. */
11544	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11545	#endif
11546	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11547
11548	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11549	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11550	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11551	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11552	{
11553	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11554	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11555	}
11556	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11557	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11558	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11559	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11560
11561	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11562	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11563	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11564	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11565	{
11566	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11567	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11568	}
11569	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11570	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11571	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11572	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11573
11574	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11575	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11576	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11577	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11578	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11579	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11580	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11581	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11582	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11583	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11584	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11585	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11586	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11587	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11588	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11589	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11590	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11591	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11592	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11593	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11594	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11595	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11596
11597	#undef IS_EITHER_ENABLED
11598	#undef SET_ONLY_XBM_IF_EITHER_EN
11599	#undef SET_CPE1_XBM_IF_EITHER_EN
11600	#undef SET_CPEU_XBM_IF_EITHER_EN
11601	#undef SET_CPE2_XBM_IF_EITHER_EN
11602
11603	/*
11604	* Sanitize the control stuff.
11605	*/
11606	pDbgState->fCpe2Extra &= g_HmMsrs.u.vmx.ProcCtls2.n.allowed1;
11607	if (pDbgState->fCpe2Extra)
11608	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11609	pDbgState->fCpe1Extra &= g_HmMsrs.u.vmx.ProcCtls.n.allowed1;
11610	pDbgState->fCpe1Unwanted &= ~g_HmMsrs.u.vmx.ProcCtls.n.allowed0;
11611	#ifndef IN_NEM_DARWIN
11612	if (pVCpu->hmr0.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11613	{
11614	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
11615	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11616	}
11617	#else
11618	if (pVCpu->nem.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11619	{
11620	pVCpu->nem.s.fDebugWantRdTscExit ^= true;
11621	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11622	}
11623	#endif
11624
11625	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11626	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11627	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11628	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11629	}
11630
11631
11632	/**
11633	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11634	* appropriate.
11635	*
11636	* The caller has checked the VM-exit against the
11637	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11638	* already, so we don't have to do that either.
11639	*
11640	* @returns Strict VBox status code (i.e. informational status codes too).
11641	* @param pVCpu The cross context virtual CPU structure.
11642	* @param pVmxTransient The VMX-transient structure.
11643	* @param uExitReason The VM-exit reason.
11644	*
11645	* @remarks The name of this function is displayed by dtrace, so keep it short
11646	* and to the point. No longer than 33 chars long, please.
11647	*/
11648	static VBOXSTRICTRC vmxHCHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11649	{
11650	/*
11651	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11652	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11653	*
11654	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11655	* does. Must add/change/remove both places. Same ordering, please.
11656	*
11657	* Added/removed events must also be reflected in the next section
11658	* where we dispatch dtrace events.
11659	*/
11660	bool fDtrace1 = false;
11661	bool fDtrace2 = false;
11662	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11663	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11664	uint32_t uEventArg = 0;
11665	#define SET_EXIT(a_EventSubName) \
11666	do { \
11667	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11668	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11669	} while (0)
11670	#define SET_BOTH(a_EventSubName) \
11671	do { \
11672	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11673	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11674	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11675	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11676	} while (0)
11677	switch (uExitReason)
11678	{
11679	case VMX_EXIT_MTF:
11680	return vmxHCExitMtf(pVCpu, pVmxTransient);
11681
11682	case VMX_EXIT_XCPT_OR_NMI:
11683	{
11684	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11685	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11686	{
11687	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11688	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11689	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11690	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11691	{
11692	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11693	{
11694	vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE>(pVCpu, pVmxTransient);
11695	uEventArg = pVmxTransient->uExitIntErrorCode;
11696	}
11697	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11698	switch (enmEvent1)
11699	{
11700	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11701	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11702	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11703	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11704	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11705	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11706	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11707	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11708	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11709	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11710	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11711	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11712	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11713	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11714	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11715	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11716	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11717	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11718	default: break;
11719	}
11720	}
11721	else
11722	AssertFailed();
11723	break;
11724
11725	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11726	uEventArg = idxVector;
11727	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11728	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11729	break;
11730	}
11731	break;
11732	}
11733
11734	case VMX_EXIT_TRIPLE_FAULT:
11735	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11736	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11737	break;
11738	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11739	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11740	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11741	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11742	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11743
11744	/* Instruction specific VM-exits: */
11745	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11746	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11747	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11748	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11749	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11750	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11751	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11752	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11753	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11754	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11755	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11756	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11757	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11758	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11759	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11760	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11761	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11762	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11763	case VMX_EXIT_MOV_CRX:
11764	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
11765	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11766	SET_BOTH(CRX_READ);
11767	else
11768	SET_BOTH(CRX_WRITE);
11769	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11770	break;
11771	case VMX_EXIT_MOV_DRX:
11772	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
11773	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11774	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11775	SET_BOTH(DRX_READ);
11776	else
11777	SET_BOTH(DRX_WRITE);
11778	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11779	break;
11780	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11781	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11782	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11783	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11784	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11785	case VMX_EXIT_GDTR_IDTR_ACCESS:
11786	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
11787	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11788	{
11789	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11790	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11791	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11792	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11793	}
11794	break;
11795
11796	case VMX_EXIT_LDTR_TR_ACCESS:
11797	vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
11798	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11799	{
11800	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11801	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11802	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11803	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11804	}
11805	break;
11806
11807	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11808	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11809	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11810	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11811	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11812	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11813	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11814	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11815	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11816	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11817	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11818
11819	/* Events that aren't relevant at this point. */
11820	case VMX_EXIT_EXT_INT:
11821	case VMX_EXIT_INT_WINDOW:
11822	case VMX_EXIT_NMI_WINDOW:
11823	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11824	case VMX_EXIT_PREEMPT_TIMER:
11825	case VMX_EXIT_IO_INSTR:
11826	break;
11827
11828	/* Errors and unexpected events. */
11829	case VMX_EXIT_INIT_SIGNAL:
11830	case VMX_EXIT_SIPI:
11831	case VMX_EXIT_IO_SMI:
11832	case VMX_EXIT_SMI:
11833	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11834	case VMX_EXIT_ERR_MSR_LOAD:
11835	case VMX_EXIT_ERR_MACHINE_CHECK:
11836	case VMX_EXIT_PML_FULL:
11837	case VMX_EXIT_VIRTUALIZED_EOI:
11838	break;
11839
11840	default:
11841	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11842	break;
11843	}
11844	#undef SET_BOTH
11845	#undef SET_EXIT
11846
11847	/*
11848	* Dtrace tracepoints go first. We do them here at once so we don't
11849	* have to copy the guest state saving and stuff a few dozen times.
11850	* Down side is that we've got to repeat the switch, though this time
11851	* we use enmEvent since the probes are a subset of what DBGF does.
11852	*/
11853	if (fDtrace1 \|\| fDtrace2)
11854	{
11855	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
11856	vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
11857	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; RT_NOREF(pCtx); /* Shut up Clang 13. */
11858	switch (enmEvent1)
11859	{
11860	/** @todo consider which extra parameters would be helpful for each probe. */
11861	case DBGFEVENT_END: break;
11862	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11863	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11864	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11865	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11866	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11867	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11868	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11869	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11870	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11871	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11872	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11873	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11874	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11875	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11876	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11877	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11878	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11879	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11880	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11881	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11882	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11883	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11884	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11885	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11886	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11887	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11888	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11889	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11890	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11891	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11892	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11893	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11894	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11895	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11896	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11897	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11898	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11899	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11900	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11901	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11902	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11903	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11904	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11905	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11906	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11907	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11908	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11909	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11910	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11911	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11912	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11913	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11914	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11915	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11916	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11917	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11918	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11919	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11920	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11921	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11922	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11923	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11924	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11925	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11926	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11927	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11928	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11929	}
11930	switch (enmEvent2)
11931	{
11932	/** @todo consider which extra parameters would be helpful for each probe. */
11933	case DBGFEVENT_END: break;
11934	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11935	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11936	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11937	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11938	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11939	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11940	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11941	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11942	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11943	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11944	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11945	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11946	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11947	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11948	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11949	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11950	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11951	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11952	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11953	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11954	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11955	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11956	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11957	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11958	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11959	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11960	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11961	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11962	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11963	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11964	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11965	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11966	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11967	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11968	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11969	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11970	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11971	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11972	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11973	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11974	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11975	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11976	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11977	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11978	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11979	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11980	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11981	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11982	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11983	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11984	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11985	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11986	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11987	}
11988	}
11989
11990	/*
11991	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11992	* the DBGF call will do a full check).
11993	*
11994	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11995	* Note! If we have to events, we prioritize the first, i.e. the instruction
11996	* one, in order to avoid event nesting.
11997	*/
11998	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
11999	if ( enmEvent1 != DBGFEVENT_END
12000	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12001	{
12002	vmxHCImportGuestState<CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
12003	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12004	if (rcStrict != VINF_SUCCESS)
12005	return rcStrict;
12006	}
12007	else if ( enmEvent2 != DBGFEVENT_END
12008	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12009	{
12010	vmxHCImportGuestState<CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
12011	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12012	if (rcStrict != VINF_SUCCESS)
12013	return rcStrict;
12014	}
12015
12016	return VINF_SUCCESS;
12017	}
12018
12019
12020	/**
12021	* Single-stepping VM-exit filtering.
12022	*
12023	* This is preprocessing the VM-exits and deciding whether we've gotten far
12024	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12025	* handling is performed.
12026	*
12027	* @returns Strict VBox status code (i.e. informational status codes too).
12028	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12029	* @param pVmxTransient The VMX-transient structure.
12030	* @param pDbgState The debug state.
12031	*/
12032	DECLINLINE(VBOXSTRICTRC) vmxHCRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12033	{
12034	/*
12035	* Expensive (saves context) generic dtrace VM-exit probe.
12036	*/
12037	uint32_t const uExitReason = pVmxTransient->uExitReason;
12038	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12039	{ /* more likely */ }
12040	else
12041	{
12042	vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
12043	int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
12044	AssertRC(rc);
12045	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12046	}
12047
12048	#ifndef IN_NEM_DARWIN
12049	/*
12050	* Check for host NMI, just to get that out of the way.
12051	*/
12052	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12053	{ /* normally likely */ }
12054	else
12055	{
12056	vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
12057	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12058	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12059	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12060	}
12061	#endif
12062
12063	/*
12064	* Check for single stepping event if we're stepping.
12065	*/
12066	if (VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
12067	{
12068	switch (uExitReason)
12069	{
12070	case VMX_EXIT_MTF:
12071	return vmxHCExitMtf(pVCpu, pVmxTransient);
12072
12073	/* Various events: */
12074	case VMX_EXIT_XCPT_OR_NMI:
12075	case VMX_EXIT_EXT_INT:
12076	case VMX_EXIT_TRIPLE_FAULT:
12077	case VMX_EXIT_INT_WINDOW:
12078	case VMX_EXIT_NMI_WINDOW:
12079	case VMX_EXIT_TASK_SWITCH:
12080	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12081	case VMX_EXIT_APIC_ACCESS:
12082	case VMX_EXIT_EPT_VIOLATION:
12083	case VMX_EXIT_EPT_MISCONFIG:
12084	case VMX_EXIT_PREEMPT_TIMER:
12085
12086	/* Instruction specific VM-exits: */
12087	case VMX_EXIT_CPUID:
12088	case VMX_EXIT_GETSEC:
12089	case VMX_EXIT_HLT:
12090	case VMX_EXIT_INVD:
12091	case VMX_EXIT_INVLPG:
12092	case VMX_EXIT_RDPMC:
12093	case VMX_EXIT_RDTSC:
12094	case VMX_EXIT_RSM:
12095	case VMX_EXIT_VMCALL:
12096	case VMX_EXIT_VMCLEAR:
12097	case VMX_EXIT_VMLAUNCH:
12098	case VMX_EXIT_VMPTRLD:
12099	case VMX_EXIT_VMPTRST:
12100	case VMX_EXIT_VMREAD:
12101	case VMX_EXIT_VMRESUME:
12102	case VMX_EXIT_VMWRITE:
12103	case VMX_EXIT_VMXOFF:
12104	case VMX_EXIT_VMXON:
12105	case VMX_EXIT_MOV_CRX:
12106	case VMX_EXIT_MOV_DRX:
12107	case VMX_EXIT_IO_INSTR:
12108	case VMX_EXIT_RDMSR:
12109	case VMX_EXIT_WRMSR:
12110	case VMX_EXIT_MWAIT:
12111	case VMX_EXIT_MONITOR:
12112	case VMX_EXIT_PAUSE:
12113	case VMX_EXIT_GDTR_IDTR_ACCESS:
12114	case VMX_EXIT_LDTR_TR_ACCESS:
12115	case VMX_EXIT_INVEPT:
12116	case VMX_EXIT_RDTSCP:
12117	case VMX_EXIT_INVVPID:
12118	case VMX_EXIT_WBINVD:
12119	case VMX_EXIT_XSETBV:
12120	case VMX_EXIT_RDRAND:
12121	case VMX_EXIT_INVPCID:
12122	case VMX_EXIT_VMFUNC:
12123	case VMX_EXIT_RDSEED:
12124	case VMX_EXIT_XSAVES:
12125	case VMX_EXIT_XRSTORS:
12126	{
12127	int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
12128	AssertRCReturn(rc, rc);
12129	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12130	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12131	return VINF_EM_DBG_STEPPED;
12132	break;
12133	}
12134
12135	/* Errors and unexpected events: */
12136	case VMX_EXIT_INIT_SIGNAL:
12137	case VMX_EXIT_SIPI:
12138	case VMX_EXIT_IO_SMI:
12139	case VMX_EXIT_SMI:
12140	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12141	case VMX_EXIT_ERR_MSR_LOAD:
12142	case VMX_EXIT_ERR_MACHINE_CHECK:
12143	case VMX_EXIT_PML_FULL:
12144	case VMX_EXIT_VIRTUALIZED_EOI:
12145	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12146	break;
12147
12148	default:
12149	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12150	break;
12151	}
12152	}
12153
12154	/*
12155	* Check for debugger event breakpoints and dtrace probes.
12156	*/
12157	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12158	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12159	{
12160	VBOXSTRICTRC rcStrict = vmxHCHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12161	if (rcStrict != VINF_SUCCESS)
12162	return rcStrict;
12163	}
12164
12165	/*
12166	* Normal processing.
12167	*/
12168	#ifdef HMVMX_USE_FUNCTION_TABLE
12169	return g_aVMExitHandlers[uExitReason].pfn(pVCpu, pVmxTransient);
12170	#else
12171	return vmxHCHandleExit(pVCpu, pVmxTransient, uExitReason);
12172	#endif
12173	}
12174
12175	/** @} */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h

Download in other formats: