HMVMXR0.cpp@ 79185

Last change on this file since 79185 was 79167, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Assert parameters in module level functions when possible and move a couple of static function parameter asserts to the caller. Nits, comments, naming consistency.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 700.4 KB

Line
1	/* $Id: HMVMXR0.cpp 79167 2019-06-17 05:32:49Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293	/** Pointer to a const VMX transient state. */
294	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
295
296
297	/**
298	* Memory operand read or write access.
299	*/
300	typedef enum VMXMEMACCESS
301	{
302	VMXMEMACCESS_READ = 0,
303	VMXMEMACCESS_WRITE = 1
304	} VMXMEMACCESS;
305
306	/**
307	* VMX VM-exit handler.
308	*
309	* @returns Strict VBox status code (i.e. informational status codes too).
310	* @param pVCpu The cross context virtual CPU structure.
311	* @param pVmxTransient The VMX-transient structure.
312	*/
313	#ifndef HMVMX_USE_FUNCTION_TABLE
314	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
315	#else
316	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
317	/** Pointer to VM-exit handler. */
318	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
319	#endif
320
321	/**
322	* VMX VM-exit handler, non-strict status code.
323	*
324	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
325	*
326	* @returns VBox status code, no informational status code returned.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pVmxTransient The VMX-transient structure.
329	*
330	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
331	* use of that status code will be replaced with VINF_EM_SOMETHING
332	* later when switching over to IEM.
333	*/
334	#ifndef HMVMX_USE_FUNCTION_TABLE
335	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
336	#else
337	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
338	#endif
339
340
341	/*********************************************************************************************************************************
342	* Internal Functions *
343	*********************************************************************************************************************************/
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
346	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
348	#else
349	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
350	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
351	#endif
352	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
353	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
354	#endif
355
356	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
357	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
358	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
359	#endif
360
361	/** @name VM-exit handlers.
362	* @{
363	*/
364	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
365	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
366	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
369	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
370	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
371	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
372	static FNVMXEXITHANDLER hmR0VmxExitHlt;
373	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
374	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
375	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
376	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
377	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
378	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
379	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
380	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
382	static FNVMXEXITHANDLER hmR0VmxExitVmread;
383	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
384	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
385	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
387	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
391	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
392	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
393	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
394	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
395	static FNVMXEXITHANDLER hmR0VmxExitMwait;
396	static FNVMXEXITHANDLER hmR0VmxExitMtf;
397	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
398	static FNVMXEXITHANDLER hmR0VmxExitPause;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
400	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
401	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
402	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
403	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
404	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
406	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
407	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
411	/** @} */
412
413	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
414	/** @name Nested-guest VM-exit handlers.
415	* @{
416	*/
417	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
418	//static FNVMXEXITHANDLER hmR0VmxExitExtIntNested;
419	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
422	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
423	//static FNVMXEXITHANDLER hmR0VmxExitCpuid;
424	//static FNVMXEXITHANDLER hmR0VmxExitGetsec;
425	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
426	//static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
427	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
428	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
429	//static FNVMXEXITHANDLER hmR0VmxExitVmcall;
430	//static FNVMXEXITHANDLER hmR0VmxExitVmclear;
431	//static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
432	//static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
433	//static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
434	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
435	//static FNVMXEXITHANDLER hmR0VmxExitVmresume;
436	//static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
437	//static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
438	//static FNVMXEXITHANDLER hmR0VmxExitVmxon;
439	//static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
440	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
441	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
442	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
443	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
444	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
445	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
446	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
447	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
448	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
449	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
450	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
451	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
452	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
453	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
454	//static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
455	//static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
456	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
457	//static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
458	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
459	//static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
460	//static FNVMXEXITHANDLER hmR0VmxExitErrUnexpected;
461	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
462	//static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
463	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
464	//static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
465	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
466	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
467	/** @} */
468	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
469
470	/** @name Helpers for hardware exceptions VM-exit handlers.
471	* @{
472	*/
473	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
474	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
475	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
476	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
477	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
478	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
479	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
480	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst);
481	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr);
482	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
483	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
484	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu);
485	/** @} */
486
487
488	/*********************************************************************************************************************************
489	* Global Variables *
490	*********************************************************************************************************************************/
491	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
492	static const uint32_t g_aVmcsCacheSegBase[] =
493	{
494	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
495	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
496	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
497	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
498	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
499	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
500	};
501	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
502	#endif
503	static const uint32_t g_aVmcsSegBase[] =
504	{
505	VMX_VMCS_GUEST_ES_BASE,
506	VMX_VMCS_GUEST_CS_BASE,
507	VMX_VMCS_GUEST_SS_BASE,
508	VMX_VMCS_GUEST_DS_BASE,
509	VMX_VMCS_GUEST_FS_BASE,
510	VMX_VMCS_GUEST_GS_BASE
511	};
512	static const uint32_t g_aVmcsSegSel[] =
513	{
514	VMX_VMCS16_GUEST_ES_SEL,
515	VMX_VMCS16_GUEST_CS_SEL,
516	VMX_VMCS16_GUEST_SS_SEL,
517	VMX_VMCS16_GUEST_DS_SEL,
518	VMX_VMCS16_GUEST_FS_SEL,
519	VMX_VMCS16_GUEST_GS_SEL
520	};
521	static const uint32_t g_aVmcsSegLimit[] =
522	{
523	VMX_VMCS32_GUEST_ES_LIMIT,
524	VMX_VMCS32_GUEST_CS_LIMIT,
525	VMX_VMCS32_GUEST_SS_LIMIT,
526	VMX_VMCS32_GUEST_DS_LIMIT,
527	VMX_VMCS32_GUEST_FS_LIMIT,
528	VMX_VMCS32_GUEST_GS_LIMIT
529	};
530	static const uint32_t g_aVmcsSegAttr[] =
531	{
532	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
533	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
534	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
535	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
536	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
537	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
538	};
539	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
540	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
541	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
542	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
543
544	#ifdef HMVMX_USE_FUNCTION_TABLE
545	/**
546	* VMX_EXIT dispatch table.
547	*/
548	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
549	{
550	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
551	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
552	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
553	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
554	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
555	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
556	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
557	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
558	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
559	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
560	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
561	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
562	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
563	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
564	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
565	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
566	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
567	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
568	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
569	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
570	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
571	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
572	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
573	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
574	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
575	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
576	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
577	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
578	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
579	#else
580	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
581	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
582	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
583	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
584	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
585	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
586	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
587	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
588	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
589	#endif
590	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
591	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
592	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
593	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
594	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
595	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
596	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
597	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
598	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
599	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
600	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
601	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
602	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
603	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
604	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
605	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
606	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
607	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
608	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
609	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
610	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
611	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
612	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
613	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
614	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
615	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
616	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
617	#else
618	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
619	#endif
620	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
621	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
622	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
623	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
624	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
625	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
626	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
627	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
628	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
629	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
630	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
631	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
632	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
633	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
634	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
635	};
636	#endif /* HMVMX_USE_FUNCTION_TABLE */
637
638	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
639	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
640	{
641	/* 0 */ "(Not Used)",
642	/* 1 */ "VMCALL executed in VMX root operation.",
643	/* 2 */ "VMCLEAR with invalid physical address.",
644	/* 3 */ "VMCLEAR with VMXON pointer.",
645	/* 4 */ "VMLAUNCH with non-clear VMCS.",
646	/* 5 */ "VMRESUME with non-launched VMCS.",
647	/* 6 */ "VMRESUME after VMXOFF",
648	/* 7 */ "VM-entry with invalid control fields.",
649	/* 8 */ "VM-entry with invalid host state fields.",
650	/* 9 */ "VMPTRLD with invalid physical address.",
651	/* 10 */ "VMPTRLD with VMXON pointer.",
652	/* 11 */ "VMPTRLD with incorrect revision identifier.",
653	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
654	/* 13 */ "VMWRITE to read-only VMCS component.",
655	/* 14 */ "(Not Used)",
656	/* 15 */ "VMXON executed in VMX root operation.",
657	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
658	/* 17 */ "VM-entry with non-launched executing VMCS.",
659	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
660	/* 19 */ "VMCALL with non-clear VMCS.",
661	/* 20 */ "VMCALL with invalid VM-exit control fields.",
662	/* 21 */ "(Not Used)",
663	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
664	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
665	/* 24 */ "VMCALL with invalid SMM-monitor features.",
666	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
667	/* 26 */ "VM-entry with events blocked by MOV SS.",
668	/* 27 */ "(Not Used)",
669	/* 28 */ "Invalid operand to INVEPT/INVVPID."
670	};
671	#endif /* VBOX_STRICT */
672
673
674	/**
675	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
676	*
677	* Any bit set in this mask is owned by the host/hypervisor and would cause a
678	* VM-exit when modified by the guest.
679	*
680	* @returns The static CR0 guest/host mask.
681	* @param pVCpu The cross context virtual CPU structure.
682	*/
683	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
684	{
685	/*
686	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
687	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
688	*/
689	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
690	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
691	* and @bugref{6944}. */
692	PVM pVM = pVCpu->CTX_SUFF(pVM);
693	return ( X86_CR0_PE
694	\| X86_CR0_NE
695	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
696	\| X86_CR0_PG
697	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
698	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
699	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
700	}
701
702
703	/**
704	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
705	*
706	* Any bit set in this mask is owned by the host/hypervisor and would cause a
707	* VM-exit when modified by the guest.
708	*
709	* @returns The static CR4 guest/host mask.
710	* @param pVCpu The cross context virtual CPU structure.
711	*/
712	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
713	{
714	/*
715	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
716	* these bits are reserved on hardware that does not support them. Since the
717	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
718	* these bits and handle it depending on whether we expose them to the guest.
719	*/
720	PVM pVM = pVCpu->CTX_SUFF(pVM);
721	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
722	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
723	return ( X86_CR4_VMXE
724	\| X86_CR4_VME
725	\| X86_CR4_PAE
726	\| X86_CR4_PGE
727	\| X86_CR4_PSE
728	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
729	\| (fPcid ? X86_CR4_PCIDE : 0));
730	}
731
732
733	/**
734	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
735	* area.
736	*
737	* @returns @c true if it's different, @c false otherwise.
738	* @param pVmcsInfo The VMCS info. object.
739	*/
740	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
741	{
742	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
743	&& pVmcsInfo->pvGuestMsrStore);
744	}
745
746
747	/**
748	* Checks whether one of the given Pin-based VM-execution controls are set.
749	*
750	* @returns @c true if set, @c false otherwise.
751	* @param pVCpu The cross context virtual CPU structure.
752	* @param pVmxTransient The VMX-transient structure.
753	* @param uPinCtls The Pin-based VM-execution controls to check.
754	*
755	* @remarks This will not check merged controls when executing a nested-guest
756	* but the original control specified by the guest hypervisor.
757	*/
758	static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
759	{
760	if (!pVmxTransient->fIsNestedGuest)
761	{
762	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
763	return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
764	}
765	return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
766	}
767
768
769	/**
770	* Checks whether one of the given Processor-based VM-execution controls are set.
771	*
772	* @returns @c true if set, @c false otherwise.
773	* @param pVCpu The cross context virtual CPU structure.
774	* @param pVmxTransient The VMX-transient structure.
775	* @param uProcCtls The Processor-based VM-execution controls to check.
776	*
777	* @remarks This will not check merged controls when executing a nested-guest
778	* but the original control specified by the guest hypervisor.
779	*/
780	static bool hmR0VmxIsProcCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
781	{
782	if (!pVmxTransient->fIsNestedGuest)
783	{
784	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
785	return RT_BOOL(pVmcsInfo->u32ProcCtls & uProcCtls);
786	}
787	return CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
788	}
789
790
791	/**
792	* Checks whether one of the given Secondary Processor-based VM-execution controls
793	* are set.
794	*
795	* @returns @c true if set, @c false otherwise.
796	* @param pVCpu The cross context virtual CPU structure.
797	* @param pVmxTransient The VMX-transient structure.
798	* @param uProcCtls2 The Secondary Processor-based VM-execution controls to
799	* check.
800	*
801	* @remarks This will not check merged controls when executing a nested-guest
802	* but the original control specified by the guest hypervisor.
803	*/
804	static bool hmR0VmxIsProcCtls2Set(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls2)
805	{
806	if (!pVmxTransient->fIsNestedGuest)
807	{
808	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
809	return RT_BOOL(pVmcsInfo->u32ProcCtls2 & uProcCtls2);
810	}
811	return CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls2);
812	}
813
814
815	#if 0
816	/**
817	* Checks whether one of the given VM-entry controls are set.
818	*
819	* @returns @c true if set, @c false otherwise.
820	* @param pVCpu The cross context virtual CPU structure.
821	* @param pVmxTransient The VMX-transient structure.
822	* @param uEntryCtls The VM-entry controls to check.
823	*
824	* @remarks This will not check merged controls when executing a nested-guest
825	* but the original control specified by the guest hypervisor.
826	*/
827	static bool hmR0VmxIsEntryCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uEntryCtls)
828	{
829	if (!pVmxTransient->fIsNestedGuest)
830	{
831	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
832	return RT_BOOL(pVmcsInfo->u32EntryCtls & uEntryCtls);
833	}
834	return CPUMIsGuestVmxEntryCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uEntryCtls);
835	}
836
837
838	/**
839	* Checks whether one of the given VM-exit controls are set.
840	*
841	* @returns @c true if set, @c false otherwise.
842	* @param pVCpu The cross context virtual CPU structure.
843	* @param pVmxTransient The VMX-transient structure.
844	* @param uExitCtls The VM-exit controls to check.
845	*
846	* @remarks This will not check merged controls when executing a nested-guest
847	* but the original control specified by the guest hypervisor.
848	*/
849	static bool hmR0VmxIsExitCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uExitCtls)
850	{
851	if (!pVmxTransient->fIsNestedGuest)
852	{
853	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
854	return RT_BOOL(pVmcsInfo->u32ExitCtls & uExitCtls);
855	}
856	return CPUMIsGuestVmxExitCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uExitCtls);
857	}
858	#endif
859
860
861	/**
862	* Adds one or more exceptions to the exception bitmap and commits it to the current
863	* VMCS.
864	*
865	* @returns VBox status code.
866	* @param pVmxTransient The VMX-transient structure.
867	* @param uXcptMask The exception(s) to add.
868	*/
869	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
870	{
871	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
872	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
873	if ((uXcptBitmap & uXcptMask) != uXcptMask)
874	{
875	uXcptBitmap \|= uXcptMask;
876	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
877	AssertRCReturn(rc, rc);
878	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
879	}
880	return VINF_SUCCESS;
881	}
882
883
884	/**
885	* Adds an exception to the exception bitmap and commits it to the current VMCS.
886	*
887	* @returns VBox status code.
888	* @param pVmxTransient The VMX-transient structure.
889	* @param uXcpt The exception to add.
890	*/
891	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
892	{
893	Assert(uXcpt <= X86_XCPT_LAST);
894	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
895	}
896
897
898	/**
899	* Remove one or more exceptions from the exception bitmap and commits it to the
900	* current VMCS.
901	*
902	* This takes care of not removing the exception intercept if a nested-guest
903	* requires the exception to be intercepted.
904	*
905	* @returns VBox status code.
906	* @param pVCpu The cross context virtual CPU structure.
907	* @param pVmxTransient The VMX-transient structure.
908	* @param uXcptMask The exception(s) to remove.
909	*/
910	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
911	{
912	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
913	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
914	if (u32XcptBitmap & uXcptMask)
915	{
916	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
917	if (!pVmxTransient->fIsNestedGuest)
918	{ /* likely */ }
919	else
920	{
921	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
922	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
923	}
924	#endif
925	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
926	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
927	\| RT_BIT(X86_XCPT_DE)
928	\| RT_BIT(X86_XCPT_NM)
929	\| RT_BIT(X86_XCPT_TS)
930	\| RT_BIT(X86_XCPT_UD)
931	\| RT_BIT(X86_XCPT_NP)
932	\| RT_BIT(X86_XCPT_SS)
933	\| RT_BIT(X86_XCPT_GP)
934	\| RT_BIT(X86_XCPT_PF)
935	\| RT_BIT(X86_XCPT_MF));
936	#elif defined(HMVMX_ALWAYS_TRAP_PF)
937	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
938	#endif
939	if (uXcptMask)
940	{
941	/* Validate we are not removing any essential exception intercepts. */
942	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
943	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
944	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
945
946	/* Remove it from the exception bitmap. */
947	u32XcptBitmap &= ~uXcptMask;
948
949	/* Commit and update the cache if necessary. */
950	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
951	{
952	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
953	AssertRCReturn(rc, rc);
954	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
955	}
956	}
957	}
958	return VINF_SUCCESS;
959	}
960
961
962	/**
963	* Remove an exceptions from the exception bitmap and commits it to the current
964	* VMCS.
965	*
966	* @returns VBox status code.
967	* @param pVCpu The cross context virtual CPU structure.
968	* @param pVmxTransient The VMX-transient structure.
969	* @param uXcpt The exception to remove.
970	*/
971	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
972	{
973	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
974	}
975
976
977	/**
978	* Loads the VMCS specified by the VMCS info. object.
979	*
980	* @returns VBox status code.
981	* @param pVmcsInfo The VMCS info. object.
982	*/
983	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
984	{
985	Assert(pVmcsInfo->HCPhysVmcs);
986	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
987
988	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
989	{
990	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
991	if (RT_SUCCESS(rc))
992	{
993	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
994	return VINF_SUCCESS;
995	}
996	return rc;
997	}
998	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
999	}
1000
1001
1002	/**
1003	* Clears the VMCS specified by the VMCS info. object.
1004	*
1005	* @returns VBox status code.
1006	* @param pVmcsInfo The VMCS info. object.
1007	*/
1008	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1009	{
1010	Assert(pVmcsInfo->HCPhysVmcs);
1011	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1012
1013	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1014	if (RT_SUCCESS(rc))
1015	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1016	return rc;
1017	}
1018
1019
1020	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1021	/**
1022	* Switches the current VMCS to the one specified.
1023	*
1024	* @returns VBox status code.
1025	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1026	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1027	*
1028	* @remarks Called with interrupts disabled.
1029	*/
1030	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1031	{
1032	/*
1033	* Clear the VMCS we are switching out if it has not already been cleared.
1034	* This will sync any CPU internal data back to the VMCS.
1035	*/
1036	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1037	{
1038	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1039	if (RT_SUCCESS(rc))
1040	{ /* likely */ }
1041	else
1042	return rc;
1043	}
1044
1045	/*
1046	* Clear the VMCS we are switching to if it has not already been cleared.
1047	* This will initialize the VMCS launch state to "clear" required for loading it.
1048	*
1049	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1050	*/
1051	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1052	{
1053	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1054	if (RT_SUCCESS(rc))
1055	{ /* likely */ }
1056	else
1057	return rc;
1058	}
1059
1060	/*
1061	* Finally, load the VMCS we are switching to.
1062	*/
1063	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1064	}
1065	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1066
1067
1068	/**
1069	* Updates the VM's last error record.
1070	*
1071	* If there was a VMX instruction error, reads the error data from the VMCS and
1072	* updates VCPU's last error record as well.
1073	*
1074	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1075	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1076	* VERR_VMX_INVALID_VMCS_FIELD.
1077	* @param rc The error code.
1078	*/
1079	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1080	{
1081	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1082	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1083	{
1084	AssertPtrReturnVoid(pVCpu);
1085	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1086	}
1087	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1088	}
1089
1090
1091	#ifdef VBOX_STRICT
1092	/**
1093	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1094	* transient structure.
1095	*
1096	* @returns VBox status code.
1097	* @param pVmxTransient The VMX-transient structure.
1098	*
1099	* @remarks No-long-jump zone!!!
1100	*/
1101	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1102	{
1103	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1104	AssertRCReturn(rc, rc);
1105	return VINF_SUCCESS;
1106	}
1107
1108
1109	/**
1110	* Reads the VM-entry exception error code field from the VMCS into
1111	* the VMX transient structure.
1112	*
1113	* @returns VBox status code.
1114	* @param pVmxTransient The VMX-transient structure.
1115	*
1116	* @remarks No-long-jump zone!!!
1117	*/
1118	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1119	{
1120	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1121	AssertRCReturn(rc, rc);
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Reads the VM-entry exception error code field from the VMCS into
1128	* the VMX transient structure.
1129	*
1130	* @returns VBox status code.
1131	* @param pVmxTransient The VMX-transient structure.
1132	*
1133	* @remarks No-long-jump zone!!!
1134	*/
1135	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1136	{
1137	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1138	AssertRCReturn(rc, rc);
1139	return VINF_SUCCESS;
1140	}
1141	#endif /* VBOX_STRICT */
1142
1143
1144	/**
1145	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1146	* transient structure.
1147	*
1148	* @returns VBox status code.
1149	* @param pVmxTransient The VMX-transient structure.
1150	*/
1151	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1152	{
1153	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1154	{
1155	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1156	AssertRCReturn(rc,rc);
1157	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1158	}
1159	return VINF_SUCCESS;
1160	}
1161
1162
1163	/**
1164	* Reads the VM-exit interruption error code from the VMCS into the VMX
1165	* transient structure.
1166	*
1167	* @returns VBox status code.
1168	* @param pVmxTransient The VMX-transient structure.
1169	*/
1170	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1171	{
1172	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1173	{
1174	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1175	AssertRCReturn(rc, rc);
1176	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1177	}
1178	return VINF_SUCCESS;
1179	}
1180
1181
1182	/**
1183	* Reads the VM-exit instruction length field from the VMCS into the VMX
1184	* transient structure.
1185	*
1186	* @returns VBox status code.
1187	* @param pVmxTransient The VMX-transient structure.
1188	*/
1189	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1190	{
1191	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1192	{
1193	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1194	AssertRCReturn(rc, rc);
1195	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1196	}
1197	return VINF_SUCCESS;
1198	}
1199
1200
1201	/**
1202	* Reads the VM-exit instruction-information field from the VMCS into
1203	* the VMX transient structure.
1204	*
1205	* @returns VBox status code.
1206	* @param pVmxTransient The VMX-transient structure.
1207	*/
1208	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1209	{
1210	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1211	{
1212	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1213	AssertRCReturn(rc, rc);
1214	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1215	}
1216	return VINF_SUCCESS;
1217	}
1218
1219
1220	/**
1221	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1222	*
1223	* @returns VBox status code.
1224	* @param pVCpu The cross context virtual CPU structure of the
1225	* calling EMT. (Required for the VMCS cache case.)
1226	* @param pVmxTransient The VMX-transient structure.
1227	*/
1228	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1229	{
1230	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1231	{
1232	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1233	AssertRCReturn(rc, rc);
1234	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1235	}
1236	return VINF_SUCCESS;
1237	}
1238
1239
1240	/**
1241	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1242	*
1243	* @returns VBox status code.
1244	* @param pVCpu The cross context virtual CPU structure of the
1245	* calling EMT. (Required for the VMCS cache case.)
1246	* @param pVmxTransient The VMX-transient structure.
1247	*/
1248	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1249	{
1250	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1251	{
1252	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1253	AssertRCReturn(rc, rc);
1254	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1255	}
1256	return VINF_SUCCESS;
1257	}
1258
1259
1260	/**
1261	* Reads the IDT-vectoring information field from the VMCS into the VMX
1262	* transient structure.
1263	*
1264	* @returns VBox status code.
1265	* @param pVmxTransient The VMX-transient structure.
1266	*
1267	* @remarks No-long-jump zone!!!
1268	*/
1269	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1270	{
1271	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1272	{
1273	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1274	AssertRCReturn(rc, rc);
1275	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1276	}
1277	return VINF_SUCCESS;
1278	}
1279
1280
1281	/**
1282	* Reads the IDT-vectoring error code from the VMCS into the VMX
1283	* transient structure.
1284	*
1285	* @returns VBox status code.
1286	* @param pVmxTransient The VMX-transient structure.
1287	*/
1288	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1289	{
1290	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1291	{
1292	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1293	AssertRCReturn(rc, rc);
1294	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1295	}
1296	return VINF_SUCCESS;
1297	}
1298
1299
1300	/**
1301	* Enters VMX root mode operation on the current CPU.
1302	*
1303	* @returns VBox status code.
1304	* @param pVM The cross context VM structure. Can be
1305	* NULL, after a resume.
1306	* @param HCPhysCpuPage Physical address of the VMXON region.
1307	* @param pvCpuPage Pointer to the VMXON region.
1308	*/
1309	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1310	{
1311	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1312	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1313	Assert(pvCpuPage);
1314	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1315
1316	if (pVM)
1317	{
1318	/* Write the VMCS revision identifier to the VMXON region. */
1319	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1320	}
1321
1322	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1323	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1324
1325	/* Enable the VMX bit in CR4 if necessary. */
1326	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1327
1328	/* Enter VMX root mode. */
1329	int rc = VMXEnable(HCPhysCpuPage);
1330	if (RT_FAILURE(rc))
1331	{
1332	if (!(uOldCr4 & X86_CR4_VMXE))
1333	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1334
1335	if (pVM)
1336	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1337	}
1338
1339	/* Restore interrupts. */
1340	ASMSetFlags(fEFlags);
1341	return rc;
1342	}
1343
1344
1345	/**
1346	* Exits VMX root mode operation on the current CPU.
1347	*
1348	* @returns VBox status code.
1349	*/
1350	static int hmR0VmxLeaveRootMode(void)
1351	{
1352	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1353
1354	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1355	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1356
1357	/* If we're for some reason not in VMX root mode, then don't leave it. */
1358	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1359
1360	int rc;
1361	if (uHostCr4 & X86_CR4_VMXE)
1362	{
1363	/* Exit VMX root mode and clear the VMX bit in CR4. */
1364	VMXDisable();
1365	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1366	rc = VINF_SUCCESS;
1367	}
1368	else
1369	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1370
1371	/* Restore interrupts. */
1372	ASMSetFlags(fEFlags);
1373	return rc;
1374	}
1375
1376
1377	/**
1378	* Allocates and maps a physically contiguous page. The allocated page is
1379	* zero'd out (used by various VT-x structures).
1380	*
1381	* @returns IPRT status code.
1382	* @param pMemObj Pointer to the ring-0 memory object.
1383	* @param ppVirt Where to store the virtual address of the
1384	* allocation.
1385	* @param pHCPhys Where to store the physical address of the
1386	* allocation.
1387	*/
1388	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1389	{
1390	AssertPtr(pMemObj);
1391	AssertPtr(ppVirt);
1392	AssertPtr(pHCPhys);
1393	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1394	if (RT_FAILURE(rc))
1395	return rc;
1396	ppVirt = RTR0MemObjAddress(pMemObj);
1397	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1398	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1399	return VINF_SUCCESS;
1400	}
1401
1402
1403	/**
1404	* Frees and unmaps an allocated, physical page.
1405	*
1406	* @param pMemObj Pointer to the ring-0 memory object.
1407	* @param ppVirt Where to re-initialize the virtual address of
1408	* allocation as 0.
1409	* @param pHCPhys Where to re-initialize the physical address of the
1410	* allocation as 0.
1411	*/
1412	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1413	{
1414	AssertPtr(pMemObj);
1415	AssertPtr(ppVirt);
1416	AssertPtr(pHCPhys);
1417	/* NULL is valid, accepted and ignored by the free function below. */
1418	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1419	*pMemObj = NIL_RTR0MEMOBJ;
1420	*ppVirt = NULL;
1421	*pHCPhys = NIL_RTHCPHYS;
1422	}
1423
1424
1425	/**
1426	* Initializes a VMCS info. object.
1427	*
1428	* @param pVmcsInfo The VMCS info. object.
1429	*/
1430	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1431	{
1432	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1433
1434	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1435	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1436	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1437	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1438	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1439	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1440	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1441	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1442	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1443	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1444	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1445	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1446	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1447	}
1448
1449
1450	/**
1451	* Frees the VT-x structures for a VMCS info. object.
1452	*
1453	* @param pVM The cross context VM structure.
1454	* @param pVmcsInfo The VMCS info. object.
1455	*/
1456	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1457	{
1458	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1459
1460	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1461	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1462
1463	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1464	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1465	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1466
1467	hmR0VmxInitVmcsInfo(pVmcsInfo);
1468	}
1469
1470
1471	/**
1472	* Allocates the VT-x structures for a VMCS info. object.
1473	*
1474	* @returns VBox status code.
1475	* @param pVCpu The cross context virtual CPU structure.
1476	* @param pVmcsInfo The VMCS info. object.
1477	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1478	*/
1479	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1480	{
1481	PVM pVM = pVCpu->CTX_SUFF(pVM);
1482
1483	/* Allocate the guest VM control structure (VMCS). */
1484	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1485	if (RT_SUCCESS(rc))
1486	{
1487	if (!fIsNstGstVmcs)
1488	{
1489	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1490	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1491	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1492	{
1493	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1494	NULL /* pR3Ptr /, NULL / pRCPtr */);
1495	}
1496	}
1497	else
1498	{
1499	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1500	Assert(!pVmcsInfo->pbVirtApic);
1501	}
1502
1503	if (RT_SUCCESS(rc))
1504	{
1505	/*
1506	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1507	* transparent accesses of specific MSRs.
1508	*
1509	* If the condition for enabling MSR bitmaps changes here, don't forget to
1510	* update HMIsMsrBitmapActive().
1511	*
1512	* We don't share MSR bitmaps between the guest and nested-guest as we then
1513	* don't need to care about carefully restoring the guest MSR bitmap.
1514	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1515	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1516	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1517	* we do that later while merging VMCS.
1518	*/
1519	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1520	{
1521	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1522	if ( RT_SUCCESS(rc)
1523	&& !fIsNstGstVmcs)
1524	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1525	}
1526
1527	if (RT_SUCCESS(rc))
1528	{
1529	/*
1530	* Allocate the VM-entry MSR-load area for the guest MSRs.
1531	*
1532	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1533	* the guest and nested-guest.
1534	*/
1535	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1536	&pVmcsInfo->HCPhysGuestMsrLoad);
1537	if (RT_SUCCESS(rc))
1538	{
1539	/*
1540	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1541	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1542	*/
1543	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1544	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1545	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1546
1547	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1548	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1549	&pVmcsInfo->HCPhysHostMsrLoad);
1550	}
1551	}
1552	}
1553	}
1554
1555	return rc;
1556	}
1557
1558
1559	/**
1560	* Free all VT-x structures for the VM.
1561	*
1562	* @returns IPRT status code.
1563	* @param pVM The cross context VM structure.
1564	*/
1565	static void hmR0VmxStructsFree(PVM pVM)
1566	{
1567	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1568	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1569	#endif
1570	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1571
1572	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1573	{
1574	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1575	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1576	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1577	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1578	if (pVM->cpum.ro.GuestFeatures.fVmx)
1579	{
1580	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1581	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1582	}
1583	#endif
1584	}
1585	}
1586
1587
1588	/**
1589	* Allocate all VT-x structures for the VM.
1590	*
1591	* @returns IPRT status code.
1592	* @param pVM The cross context VM structure.
1593	*/
1594	static int hmR0VmxStructsAlloc(PVM pVM)
1595	{
1596	/*
1597	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1598	* The VMCS size cannot be more than 4096 bytes.
1599	*
1600	* See Intel spec. Appendix A.1 "Basic VMX Information".
1601	*/
1602	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1603	if (cbVmcs <= X86_PAGE_4K_SIZE)
1604	{ /* likely */ }
1605	else
1606	{
1607	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1608	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1609	}
1610
1611	/*
1612	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1613	*/
1614	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1615	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1616	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1617	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1618	#endif
1619
1620	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1621	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1622	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1623
1624	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1625	{
1626	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1627	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1628	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1629	}
1630
1631	/*
1632	* Allocate per-VM VT-x structures.
1633	*/
1634	int rc = VINF_SUCCESS;
1635	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1636	/* Allocate crash-dump magic scratch page. */
1637	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1638	if (RT_FAILURE(rc))
1639	{
1640	hmR0VmxStructsFree(pVM);
1641	return rc;
1642	}
1643	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1644	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1645	#endif
1646
1647	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1648	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1649	{
1650	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1651	&pVM->hm.s.vmx.HCPhysApicAccess);
1652	if (RT_FAILURE(rc))
1653	{
1654	hmR0VmxStructsFree(pVM);
1655	return rc;
1656	}
1657	}
1658
1659	/*
1660	* Initialize per-VCPU VT-x structures.
1661	*/
1662	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1663	{
1664	/* Allocate the guest VMCS structures. */
1665	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1666	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1667	if (RT_SUCCESS(rc))
1668	{
1669	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1670	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1671	if (pVM->cpum.ro.GuestFeatures.fVmx)
1672	{
1673	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1674	if (RT_SUCCESS(rc))
1675	{ /* likely */ }
1676	else
1677	break;
1678	}
1679	#endif
1680	}
1681	else
1682	break;
1683	}
1684
1685	if (RT_FAILURE(rc))
1686	{
1687	hmR0VmxStructsFree(pVM);
1688	return rc;
1689	}
1690
1691	return VINF_SUCCESS;
1692	}
1693
1694
1695	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1696	/**
1697	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1698	*
1699	* @returns @c true if the MSR is intercepted, @c false otherwise.
1700	* @param pvMsrBitmap The MSR bitmap.
1701	* @param offMsr The MSR byte offset.
1702	* @param iBit The bit offset from the byte offset.
1703	*/
1704	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1705	{
1706	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1707	Assert(pbMsrBitmap);
1708	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1709	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1710	}
1711	#endif
1712
1713
1714	/**
1715	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1716	*
1717	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1718	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1719	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1720	* the read/write access of this MSR.
1721	*
1722	* @param pVCpu The cross context virtual CPU structure.
1723	* @param pVmcsInfo The VMCS info. object.
1724	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1725	* @param idMsr The MSR value.
1726	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1727	* include both a read -and- a write permission!
1728	*
1729	* @sa CPUMGetVmxMsrPermission.
1730	* @remarks Can be called with interrupts disabled.
1731	*/
1732	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1733	{
1734	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1735	Assert(pbMsrBitmap);
1736	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1737
1738	/*
1739	* MSR-bitmap Layout:
1740	* Byte index MSR range Interpreted as
1741	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1742	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1743	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1744	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1745	*
1746	* A bit corresponding to an MSR within the above range causes a VM-exit
1747	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1748	* the MSR range, it always cause a VM-exit.
1749	*
1750	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1751	*/
1752	uint16_t const offBitmapRead = 0;
1753	uint16_t const offBitmapWrite = 0x800;
1754	uint16_t offMsr;
1755	int32_t iBit;
1756	if (idMsr <= UINT32_C(0x00001fff))
1757	{
1758	offMsr = 0;
1759	iBit = idMsr;
1760	}
1761	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1762	{
1763	offMsr = 0x400;
1764	iBit = idMsr - UINT32_C(0xc0000000);
1765	}
1766	else
1767	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1768
1769	/*
1770	* Set the MSR read permission.
1771	*/
1772	uint16_t const offMsrRead = offBitmapRead + offMsr;
1773	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1774	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1775	{
1776	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1777	bool const fClear = !fIsNstGstVmcs ? true
1778	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1779	#else
1780	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1781	bool const fClear = true;
1782	#endif
1783	if (fClear)
1784	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1785	}
1786	else
1787	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1788
1789	/*
1790	* Set the MSR write permission.
1791	*/
1792	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1793	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1794	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1795	{
1796	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1797	bool const fClear = !fIsNstGstVmcs ? true
1798	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1799	#else
1800	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1801	bool const fClear = true;
1802	#endif
1803	if (fClear)
1804	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1805	}
1806	else
1807	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1808	}
1809
1810
1811	/**
1812	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1813	* area.
1814	*
1815	* @returns VBox status code.
1816	* @param pVCpu The cross context virtual CPU structure.
1817	* @param pVmcsInfo The VMCS info. object.
1818	* @param cMsrs The number of MSRs.
1819	*/
1820	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1821	{
1822	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1823	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1824	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1825	{
1826	/* Commit the MSR counts to the VMCS and update the cache. */
1827	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1828	{
1829	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1830	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1831	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1832	AssertRCReturn(rc, rc);
1833
1834	pVmcsInfo->cEntryMsrLoad = cMsrs;
1835	pVmcsInfo->cExitMsrStore = cMsrs;
1836	pVmcsInfo->cExitMsrLoad = cMsrs;
1837	}
1838	return VINF_SUCCESS;
1839	}
1840
1841	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1842	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1843	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1844	}
1845
1846
1847	/**
1848	* Adds a new (or updates the value of an existing) guest/host MSR
1849	* pair to be swapped during the world-switch as part of the
1850	* auto-load/store MSR area in the VMCS.
1851	*
1852	* @returns VBox status code.
1853	* @param pVCpu The cross context virtual CPU structure.
1854	* @param pVmxTransient The VMX-transient structure.
1855	* @param idMsr The MSR.
1856	* @param uGuestMsrValue Value of the guest MSR.
1857	* @param fSetReadWrite Whether to set the guest read/write access of this
1858	* MSR (thus not causing a VM-exit).
1859	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1860	* necessary.
1861	*/
1862	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1863	bool fSetReadWrite, bool fUpdateHostMsr)
1864	{
1865	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1866	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1867	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1868	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1869	uint32_t i;
1870
1871	/* Paranoia. */
1872	Assert(pGuestMsrLoad);
1873
1874	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1875
1876	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1877	for (i = 0; i < cMsrs; i++)
1878	{
1879	if (pGuestMsrLoad[i].u32Msr == idMsr)
1880	break;
1881	}
1882
1883	bool fAdded = false;
1884	if (i == cMsrs)
1885	{
1886	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1887	++cMsrs;
1888	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1889	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1890
1891	/* Set the guest to read/write this MSR without causing VM-exits. */
1892	if ( fSetReadWrite
1893	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1894	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1895
1896	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1897	fAdded = true;
1898	}
1899
1900	/* Update the MSR value for the newly added or already existing MSR. */
1901	pGuestMsrLoad[i].u32Msr = idMsr;
1902	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1903
1904	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1905	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1906	{
1907	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1908	pGuestMsrStore[i].u32Msr = idMsr;
1909	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1910	}
1911
1912	/* Update the corresponding slot in the host MSR area. */
1913	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1914	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1915	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1916	pHostMsr[i].u32Msr = idMsr;
1917
1918	/*
1919	* Only if the caller requests to update the host MSR value AND we've newly added the
1920	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1921	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1922	*
1923	* We do this for performance reasons since reading MSRs may be quite expensive.
1924	*/
1925	if (fAdded)
1926	{
1927	if (fUpdateHostMsr)
1928	{
1929	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1930	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1931	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1932	}
1933	else
1934	{
1935	/* Someone else can do the work. */
1936	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1937	}
1938	}
1939	return VINF_SUCCESS;
1940	}
1941
1942
1943	/**
1944	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1945	* auto-load/store MSR area in the VMCS.
1946	*
1947	* @returns VBox status code.
1948	* @param pVCpu The cross context virtual CPU structure.
1949	* @param pVmxTransient The VMX-transient structure.
1950	* @param idMsr The MSR.
1951	*/
1952	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1953	{
1954	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1955	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1956	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1957	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1958
1959	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1960
1961	for (uint32_t i = 0; i < cMsrs; i++)
1962	{
1963	/* Find the MSR. */
1964	if (pGuestMsrLoad[i].u32Msr == idMsr)
1965	{
1966	/*
1967	* If it's the last MSR, we only need to reduce the MSR count.
1968	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1969	*/
1970	if (i < cMsrs - 1)
1971	{
1972	/* Remove it from the VM-entry MSR-load area. */
1973	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1974	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1975
1976	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1977	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1978	{
1979	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1980	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1981	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1982	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1983	}
1984
1985	/* Remove it from the VM-exit MSR-load area. */
1986	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1987	Assert(pHostMsr[i].u32Msr == idMsr);
1988	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1989	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1990	}
1991
1992	/* Reduce the count to reflect the removed MSR and bail. */
1993	--cMsrs;
1994	break;
1995	}
1996	}
1997
1998	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1999	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2000	{
2001	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2002	AssertRCReturn(rc, rc);
2003
2004	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2005	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2006	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2007
2008	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2009	return VINF_SUCCESS;
2010	}
2011
2012	return VERR_NOT_FOUND;
2013	}
2014
2015
2016	/**
2017	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2018	*
2019	* @returns @c true if found, @c false otherwise.
2020	* @param pVmcsInfo The VMCS info. object.
2021	* @param idMsr The MSR to find.
2022	*/
2023	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2024	{
2025	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2026	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2027	Assert(pMsrs);
2028	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2029	for (uint32_t i = 0; i < cMsrs; i++)
2030	{
2031	if (pMsrs[i].u32Msr == idMsr)
2032	return true;
2033	}
2034	return false;
2035	}
2036
2037
2038	/**
2039	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2040	*
2041	* @param pVCpu The cross context virtual CPU structure.
2042	* @param pVmcsInfo The VMCS info. object.
2043	*
2044	* @remarks No-long-jump zone!!!
2045	*/
2046	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2047	{
2048	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2049
2050	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2051	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2052	Assert(pHostMsrLoad);
2053	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2054	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2055	for (uint32_t i = 0; i < cMsrs; i++)
2056	{
2057	/*
2058	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2059	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2060	*/
2061	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2062	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2063	else
2064	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2065	}
2066	}
2067
2068
2069	/**
2070	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2071	* perform lazy restoration of the host MSRs while leaving VT-x.
2072	*
2073	* @param pVCpu The cross context virtual CPU structure.
2074	*
2075	* @remarks No-long-jump zone!!!
2076	*/
2077	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2078	{
2079	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2080
2081	/*
2082	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2083	*/
2084	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2085	{
2086	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2087	#if HC_ARCH_BITS == 64
2088	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2089	{
2090	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2091	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2092	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2093	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2094	}
2095	#endif
2096	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2097	}
2098	}
2099
2100
2101	/**
2102	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2103	* lazily while leaving VT-x.
2104	*
2105	* @returns true if it does, false otherwise.
2106	* @param pVCpu The cross context virtual CPU structure.
2107	* @param idMsr The MSR to check.
2108	*/
2109	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2110	{
2111	NOREF(pVCpu);
2112	#if HC_ARCH_BITS == 64
2113	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2114	{
2115	switch (idMsr)
2116	{
2117	case MSR_K8_LSTAR:
2118	case MSR_K6_STAR:
2119	case MSR_K8_SF_MASK:
2120	case MSR_K8_KERNEL_GS_BASE:
2121	return true;
2122	}
2123	}
2124	#else
2125	RT_NOREF(pVCpu, idMsr);
2126	#endif
2127	return false;
2128	}
2129
2130
2131	/**
2132	* Loads a set of guests MSRs to allow read/passthru to the guest.
2133	*
2134	* The name of this function is slightly confusing. This function does NOT
2135	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2136	* common prefix for functions dealing with "lazy restoration" of the shared
2137	* MSRs.
2138	*
2139	* @param pVCpu The cross context virtual CPU structure.
2140	*
2141	* @remarks No-long-jump zone!!!
2142	*/
2143	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2144	{
2145	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2146	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2147
2148	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2149	#if HC_ARCH_BITS == 64
2150	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2151	{
2152	/*
2153	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2154	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2155	* we can skip a few MSR writes.
2156	*
2157	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2158	* guest MSR values in the guest-CPU context might be different to what's currently
2159	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2160	* CPU, see @bugref{8728}.
2161	*/
2162	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2163	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2164	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2165	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2166	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2167	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2168	{
2169	#ifdef VBOX_STRICT
2170	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2171	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2172	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2173	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2174	#endif
2175	}
2176	else
2177	{
2178	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2179	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2180	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2181	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2182	}
2183	}
2184	#endif
2185	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2186	}
2187
2188
2189	/**
2190	* Performs lazy restoration of the set of host MSRs if they were previously
2191	* loaded with guest MSR values.
2192	*
2193	* @param pVCpu The cross context virtual CPU structure.
2194	*
2195	* @remarks No-long-jump zone!!!
2196	* @remarks The guest MSRs should have been saved back into the guest-CPU
2197	* context by hmR0VmxImportGuestState()!!!
2198	*/
2199	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2200	{
2201	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2202	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2203
2204	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2205	{
2206	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2207	#if HC_ARCH_BITS == 64
2208	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2209	{
2210	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2211	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2212	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2213	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2214	}
2215	#endif
2216	}
2217	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2218	}
2219
2220
2221	/**
2222	* Verifies that our cached values of the VMCS fields are all consistent with
2223	* what's actually present in the VMCS.
2224	*
2225	* @returns VBox status code.
2226	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2227	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2228	* VMCS content. HMCPU error-field is
2229	* updated, see VMX_VCI_XXX.
2230	* @param pVCpu The cross context virtual CPU structure.
2231	* @param pVmcsInfo The VMCS info. object.
2232	*/
2233	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2234	{
2235	uint32_t u32Val;
2236	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2237	AssertRCReturn(rc, rc);
2238	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2239	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2240	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2241	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2242
2243	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2244	AssertRCReturn(rc, rc);
2245	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2246	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2247	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2248	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2249
2250	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2251	AssertRCReturn(rc, rc);
2252	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2253	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2254	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2255	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2256
2257	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2258	AssertRCReturn(rc, rc);
2259	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2260	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2261	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2262	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2263
2264	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2265	{
2266	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2267	AssertRCReturn(rc, rc);
2268	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2269	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2270	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2271	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2272	}
2273
2274	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2275	AssertRCReturn(rc, rc);
2276	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2277	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2278	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2279	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2280
2281	uint64_t u64Val;
2282	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2283	AssertRCReturn(rc, rc);
2284	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2285	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2286	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2287	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2288
2289	return VINF_SUCCESS;
2290	}
2291
2292
2293	#ifdef VBOX_STRICT
2294	/**
2295	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2296	*
2297	* @param pVCpu The cross context virtual CPU structure.
2298	* @param pVmcsInfo The VMCS info. object.
2299	*/
2300	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2301	{
2302	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2303
2304	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2305	{
2306	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2307	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2308	uint64_t uVmcsEferMsrVmcs;
2309	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2310	AssertRC(rc);
2311
2312	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2313	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2314	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2315	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2316	}
2317	}
2318
2319
2320	/**
2321	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2322	* VMCS are correct.
2323	*
2324	* @param pVCpu The cross context virtual CPU structure.
2325	* @param pVmcsInfo The VMCS info. object.
2326	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2327	*/
2328	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2329	{
2330	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2331
2332	/* Read the various MSR-area counts from the VMCS. */
2333	uint32_t cEntryLoadMsrs;
2334	uint32_t cExitStoreMsrs;
2335	uint32_t cExitLoadMsrs;
2336	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2337	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2338	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2339
2340	/* Verify all the MSR counts are the same. */
2341	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2342	Assert(cExitStoreMsrs == cExitLoadMsrs);
2343	uint32_t const cMsrs = cExitLoadMsrs;
2344
2345	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2346	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2347
2348	/* Verify the MSR counts are within the allocated page size. */
2349	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2350
2351	/* Verify the relevant contents of the MSR areas match. */
2352	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2353	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2354	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2355	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2356	for (uint32_t i = 0; i < cMsrs; i++)
2357	{
2358	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2359	if (fSeparateExitMsrStorePage)
2360	{
2361	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2362	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2363	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2364	}
2365
2366	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2367	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2368	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2369
2370	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2371	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2372	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2373	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2374
2375	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2376	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2377	if (fIsEferMsr)
2378	{
2379	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2380	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2381	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2382	}
2383
2384	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2385	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2386	{
2387	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2388	if (fIsEferMsr)
2389	{
2390	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2391	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2392	}
2393	else
2394	{
2395	if (!fIsNstGstVmcs)
2396	{
2397	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2398	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2399	}
2400	else
2401	{
2402	/*
2403	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2404	* execute a nested-guest with MSR passthrough.
2405	*
2406	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2407	* allow passthrough too.
2408	*/
2409	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2410	Assert(pvMsrBitmapNstGst);
2411	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2412	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2413	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2414	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2415	}
2416	}
2417	}
2418
2419	/* Move to the next MSR. */
2420	pHostMsrLoad++;
2421	pGuestMsrLoad++;
2422	pGuestMsrStore++;
2423	}
2424	}
2425	#endif /* VBOX_STRICT */
2426
2427
2428	/**
2429	* Flushes the TLB using EPT.
2430	*
2431	* @returns VBox status code.
2432	* @param pVCpu The cross context virtual CPU structure of the calling
2433	* EMT. Can be NULL depending on @a enmTlbFlush.
2434	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2435	* enmTlbFlush.
2436	* @param enmTlbFlush Type of flush.
2437	*
2438	* @remarks Caller is responsible for making sure this function is called only
2439	* when NestedPaging is supported and providing @a enmTlbFlush that is
2440	* supported by the CPU.
2441	* @remarks Can be called with interrupts disabled.
2442	*/
2443	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2444	{
2445	uint64_t au64Descriptor[2];
2446	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2447	au64Descriptor[0] = 0;
2448	else
2449	{
2450	Assert(pVCpu);
2451	Assert(pVmcsInfo);
2452	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2453	}
2454	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2455
2456	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2457	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2458
2459	if ( RT_SUCCESS(rc)
2460	&& pVCpu)
2461	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2462	}
2463
2464
2465	/**
2466	* Flushes the TLB using VPID.
2467	*
2468	* @returns VBox status code.
2469	* @param pVCpu The cross context virtual CPU structure of the calling
2470	* EMT. Can be NULL depending on @a enmTlbFlush.
2471	* @param enmTlbFlush Type of flush.
2472	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2473	* on @a enmTlbFlush).
2474	*
2475	* @remarks Can be called with interrupts disabled.
2476	*/
2477	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2478	{
2479	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2480
2481	uint64_t au64Descriptor[2];
2482	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2483	{
2484	au64Descriptor[0] = 0;
2485	au64Descriptor[1] = 0;
2486	}
2487	else
2488	{
2489	AssertPtr(pVCpu);
2490	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2491	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2492	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2493	au64Descriptor[1] = GCPtr;
2494	}
2495
2496	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2497	AssertMsg(rc == VINF_SUCCESS,
2498	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2499
2500	if ( RT_SUCCESS(rc)
2501	&& pVCpu)
2502	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2503	NOREF(rc);
2504	}
2505
2506
2507	/**
2508	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2509	* otherwise there is nothing really to invalidate.
2510	*
2511	* @returns VBox status code.
2512	* @param pVCpu The cross context virtual CPU structure.
2513	* @param GCVirt Guest virtual address of the page to invalidate.
2514	*/
2515	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2516	{
2517	AssertPtr(pVCpu);
2518	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2519
2520	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2521	{
2522	/*
2523	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2524	* the EPT case. See @bugref{6043} and @bugref{6177}.
2525	*
2526	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2527	* as this function maybe called in a loop with individual addresses.
2528	*/
2529	PVM pVM = pVCpu->CTX_SUFF(pVM);
2530	if (pVM->hm.s.vmx.fVpid)
2531	{
2532	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2533
2534	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2535	/*
2536	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2537	* where executing INVVPID outside 64-bit mode does not flush translations of
2538	* 64-bit linear addresses, see @bugref{6208#c72}.
2539	*/
2540	if (RT_HI_U32(GCVirt))
2541	fVpidFlush = false;
2542	#endif
2543
2544	if (fVpidFlush)
2545	{
2546	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2547	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2548	}
2549	else
2550	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2551	}
2552	else if (pVM->hm.s.fNestedPaging)
2553	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2554	}
2555
2556	return VINF_SUCCESS;
2557	}
2558
2559
2560	/**
2561	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2562	* case where neither EPT nor VPID is supported by the CPU.
2563	*
2564	* @param pHostCpu The HM physical-CPU structure.
2565	* @param pVCpu The cross context virtual CPU structure.
2566	*
2567	* @remarks Called with interrupts disabled.
2568	*/
2569	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2570	{
2571	AssertPtr(pVCpu);
2572	AssertPtr(pHostCpu);
2573
2574	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2575
2576	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2577	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2578	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2579	pVCpu->hm.s.fForceTLBFlush = false;
2580	return;
2581	}
2582
2583
2584	/**
2585	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2586	*
2587	* @param pHostCpu The HM physical-CPU structure.
2588	* @param pVCpu The cross context virtual CPU structure.
2589	* @param pVmcsInfo The VMCS info. object.
2590	*
2591	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2592	* nomenclature. The reason is, to avoid confusion in compare statements
2593	* since the host-CPU copies are named "ASID".
2594	*
2595	* @remarks Called with interrupts disabled.
2596	*/
2597	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2598	{
2599	#ifdef VBOX_WITH_STATISTICS
2600	bool fTlbFlushed = false;
2601	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2602	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2603	if (!fTlbFlushed) \
2604	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2605	} while (0)
2606	#else
2607	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2608	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2609	#endif
2610
2611	AssertPtr(pVCpu);
2612	AssertPtr(pHostCpu);
2613	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2614
2615	PVM pVM = pVCpu->CTX_SUFF(pVM);
2616	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2617	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2618	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2619
2620	/*
2621	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2622	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2623	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2624	* cannot reuse the current ASID anymore.
2625	*/
2626	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2627	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2628	{
2629	++pHostCpu->uCurrentAsid;
2630	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2631	{
2632	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2633	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2634	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2635	}
2636
2637	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2638	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2639	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2640
2641	/*
2642	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2643	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2644	*/
2645	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2646	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2647	HMVMX_SET_TAGGED_TLB_FLUSHED();
2648	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2649	}
2650	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2651	{
2652	/*
2653	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2654	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2655	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2656	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2657	* mappings, see @bugref{6568}.
2658	*
2659	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2660	*/
2661	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2662	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2663	HMVMX_SET_TAGGED_TLB_FLUSHED();
2664	}
2665
2666	pVCpu->hm.s.fForceTLBFlush = false;
2667	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2668
2669	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2670	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2671	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2672	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2673	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2674	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2675	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2676	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2677	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2678
2679	/* Update VMCS with the VPID. */
2680	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2681	AssertRC(rc);
2682
2683	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2684	}
2685
2686
2687	/**
2688	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2689	*
2690	* @param pHostCpu The HM physical-CPU structure.
2691	* @param pVCpu The cross context virtual CPU structure.
2692	* @param pVmcsInfo The VMCS info. object.
2693	*
2694	* @remarks Called with interrupts disabled.
2695	*/
2696	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2697	{
2698	AssertPtr(pVCpu);
2699	AssertPtr(pHostCpu);
2700	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2701	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2702	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2703
2704	/*
2705	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2706	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2707	*/
2708	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2709	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2710	{
2711	pVCpu->hm.s.fForceTLBFlush = true;
2712	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2713	}
2714
2715	/* Check for explicit TLB flushes. */
2716	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2717	{
2718	pVCpu->hm.s.fForceTLBFlush = true;
2719	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2720	}
2721
2722	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2723	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2724
2725	if (pVCpu->hm.s.fForceTLBFlush)
2726	{
2727	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2728	pVCpu->hm.s.fForceTLBFlush = false;
2729	}
2730	}
2731
2732
2733	/**
2734	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2735	*
2736	* @param pHostCpu The HM physical-CPU structure.
2737	* @param pVCpu The cross context virtual CPU structure.
2738	*
2739	* @remarks Called with interrupts disabled.
2740	*/
2741	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2742	{
2743	AssertPtr(pVCpu);
2744	AssertPtr(pHostCpu);
2745	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2746	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2747	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2748
2749	/*
2750	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2751	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2752	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2753	* cannot reuse the current ASID anymore.
2754	*/
2755	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2756	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2757	{
2758	pVCpu->hm.s.fForceTLBFlush = true;
2759	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2760	}
2761
2762	/* Check for explicit TLB flushes. */
2763	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2764	{
2765	/*
2766	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2767	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2768	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2769	* include fExplicitFlush's too) - an obscure corner case.
2770	*/
2771	pVCpu->hm.s.fForceTLBFlush = true;
2772	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2773	}
2774
2775	PVM pVM = pVCpu->CTX_SUFF(pVM);
2776	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2777	if (pVCpu->hm.s.fForceTLBFlush)
2778	{
2779	++pHostCpu->uCurrentAsid;
2780	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2781	{
2782	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2783	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2784	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2785	}
2786
2787	pVCpu->hm.s.fForceTLBFlush = false;
2788	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2789	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2790	if (pHostCpu->fFlushAsidBeforeUse)
2791	{
2792	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2793	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2794	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2795	{
2796	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2797	pHostCpu->fFlushAsidBeforeUse = false;
2798	}
2799	else
2800	{
2801	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2802	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2803	}
2804	}
2805	}
2806
2807	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2808	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2809	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2810	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2811	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2812	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2813	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2814
2815	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2816	AssertRC(rc);
2817	}
2818
2819
2820	/**
2821	* Flushes the guest TLB entry based on CPU capabilities.
2822	*
2823	* @param pHostCpu The HM physical-CPU structure.
2824	* @param pVCpu The cross context virtual CPU structure.
2825	* @param pVmcsInfo The VMCS info. object.
2826	*
2827	* @remarks Called with interrupts disabled.
2828	*/
2829	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2830	{
2831	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2832	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2833	#endif
2834	PVM pVM = pVCpu->CTX_SUFF(pVM);
2835	switch (pVM->hm.s.vmx.enmTlbFlushType)
2836	{
2837	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2838	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2839	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2840	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2841	default:
2842	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2843	break;
2844	}
2845	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2846	}
2847
2848
2849	/**
2850	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2851	* TLB entries from the host TLB before VM-entry.
2852	*
2853	* @returns VBox status code.
2854	* @param pVM The cross context VM structure.
2855	*/
2856	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2857	{
2858	/*
2859	* Determine optimal flush type for nested paging.
2860	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2861	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2862	*/
2863	if (pVM->hm.s.fNestedPaging)
2864	{
2865	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2866	{
2867	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2868	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2869	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2870	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2871	else
2872	{
2873	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2874	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2875	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2876	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2877	}
2878
2879	/* Make sure the write-back cacheable memory type for EPT is supported. */
2880	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2881	{
2882	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2883	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2885	}
2886
2887	/* EPT requires a page-walk length of 4. */
2888	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2889	{
2890	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2891	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2892	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2893	}
2894	}
2895	else
2896	{
2897	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2898	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2899	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2900	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2901	}
2902	}
2903
2904	/*
2905	* Determine optimal flush type for VPID.
2906	*/
2907	if (pVM->hm.s.vmx.fVpid)
2908	{
2909	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2910	{
2911	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2912	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2913	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2914	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2915	else
2916	{
2917	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2918	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2919	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2920	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2921	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2922	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2923	pVM->hm.s.vmx.fVpid = false;
2924	}
2925	}
2926	else
2927	{
2928	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2929	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2930	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2931	pVM->hm.s.vmx.fVpid = false;
2932	}
2933	}
2934
2935	/*
2936	* Setup the handler for flushing tagged-TLBs.
2937	*/
2938	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2939	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2940	else if (pVM->hm.s.fNestedPaging)
2941	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2942	else if (pVM->hm.s.vmx.fVpid)
2943	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2944	else
2945	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2946	return VINF_SUCCESS;
2947	}
2948
2949
2950	/**
2951	* Sets up the virtual-APIC page address for the VMCS.
2952	*
2953	* @returns VBox status code.
2954	* @param pVCpu The cross context virtual CPU structure.
2955	* @param pVmcsInfo The VMCS info. object.
2956	*/
2957	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2958	{
2959	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2960	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2961	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2962	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2963	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2964	}
2965
2966
2967	/**
2968	* Sets up the MSR-bitmap address for the VMCS.
2969	*
2970	* @returns VBox status code.
2971	* @param pVCpu The cross context virtual CPU structure.
2972	* @param pVmcsInfo The VMCS info. object.
2973	*/
2974	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2975	{
2976	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2977	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2978	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2979	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2980	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2981	}
2982
2983
2984	/**
2985	* Sets up the APIC-access page address for the VMCS.
2986	*
2987	* @returns VBox status code.
2988	* @param pVCpu The cross context virtual CPU structure.
2989	*/
2990	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2991	{
2992	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2993	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2994	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2995	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2996	}
2997
2998
2999	/**
3000	* Sets up the VMCS link pointer for the VMCS.
3001	*
3002	* @returns VBox status code.
3003	* @param pVCpu The cross context virtual CPU structure.
3004	* @param pVmcsInfo The VMCS info. object.
3005	*/
3006	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3007	{
3008	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3009	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
3010	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
3011	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
3012	}
3013
3014
3015	/**
3016	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3017	* in the VMCS.
3018	*
3019	* @returns VBox status code.
3020	* @param pVCpu The cross context virtual CPU structure.
3021	* @param pVmcsInfo The VMCS info. object.
3022	*/
3023	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3024	{
3025	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3026
3027	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3028	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3029	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3030
3031	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3032	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3033	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3034
3035	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3036	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3037	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3038
3039	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3040	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3041	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3042	AssertRCReturn(rc, rc);
3043	return VINF_SUCCESS;
3044	}
3045
3046
3047	/**
3048	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3049	*
3050	* @param pVCpu The cross context virtual CPU structure.
3051	* @param pVmcsInfo The VMCS info. object.
3052	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3053	*/
3054	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3055	{
3056	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3057
3058	/*
3059	* The guest can access the following MSRs (read, write) without causing
3060	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3061	*/
3062	PVM pVM = pVCpu->CTX_SUFF(pVM);
3063	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3064	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3065	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3066	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3067	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3068
3069	/*
3070	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3071	* associated with then. We never need to intercept access (writes need to be
3072	* executed without causing a VM-exit, reads will #GP fault anyway).
3073	*
3074	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3075	* read/write them. We swap the the guest/host MSR value using the
3076	* auto-load/store MSR area.
3077	*/
3078	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3079	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3080	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3081	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3082	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3083	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3084
3085	#if HC_ARCH_BITS == 64
3086	/*
3087	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3088	* required for 64-bit guests.
3089	*/
3090	if (pVM->hm.s.fAllow64BitGuests)
3091	{
3092	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3093	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3094	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3095	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3096	}
3097	#endif
3098
3099	/*
3100	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3101	*/
3102	#ifdef VBOX_STRICT
3103	Assert(pVmcsInfo->pvMsrBitmap);
3104	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3105	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3106	#endif
3107	}
3108
3109
3110	/**
3111	* Sets up pin-based VM-execution controls in the VMCS.
3112	*
3113	* @returns VBox status code.
3114	* @param pVCpu The cross context virtual CPU structure.
3115	* @param pVmcsInfo The VMCS info. object.
3116	*/
3117	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3118	{
3119	PVM pVM = pVCpu->CTX_SUFF(pVM);
3120	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3121	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3122
3123	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3124	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3125
3126	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3127	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3128
3129	/* Enable the VMX-preemption timer. */
3130	if (pVM->hm.s.vmx.fUsePreemptTimer)
3131	{
3132	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3133	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3134	}
3135
3136	#if 0
3137	/* Enable posted-interrupt processing. */
3138	if (pVM->hm.s.fPostedIntrs)
3139	{
3140	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3141	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3142	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3143	}
3144	#endif
3145
3146	if ((fVal & fZap) != fVal)
3147	{
3148	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3149	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3150	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3151	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3152	}
3153
3154	/* Commit it to the VMCS and update our cache. */
3155	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3156	AssertRCReturn(rc, rc);
3157	pVmcsInfo->u32PinCtls = fVal;
3158
3159	return VINF_SUCCESS;
3160	}
3161
3162
3163	/**
3164	* Sets up secondary processor-based VM-execution controls in the VMCS.
3165	*
3166	* @returns VBox status code.
3167	* @param pVCpu The cross context virtual CPU structure.
3168	* @param pVmcsInfo The VMCS info. object.
3169	*/
3170	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3171	{
3172	PVM pVM = pVCpu->CTX_SUFF(pVM);
3173	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3174	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3175
3176	/* WBINVD causes a VM-exit. */
3177	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3178	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3179
3180	/* Enable EPT (aka nested-paging). */
3181	if (pVM->hm.s.fNestedPaging)
3182	fVal \|= VMX_PROC_CTLS2_EPT;
3183
3184	/* Enable the INVPCID instruction if supported by the hardware and we expose
3185	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
3186	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
3187	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
3188	fVal \|= VMX_PROC_CTLS2_INVPCID;
3189
3190	/* Enable VPID. */
3191	if (pVM->hm.s.vmx.fVpid)
3192	fVal \|= VMX_PROC_CTLS2_VPID;
3193
3194	/* Enable unrestricted guest execution. */
3195	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3196	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3197
3198	#if 0
3199	if (pVM->hm.s.fVirtApicRegs)
3200	{
3201	/* Enable APIC-register virtualization. */
3202	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3203	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3204
3205	/* Enable virtual-interrupt delivery. */
3206	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3207	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3208	}
3209	#endif
3210
3211	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3212	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3213	* done dynamically. */
3214	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3215	{
3216	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3217	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3218	AssertRCReturn(rc, rc);
3219	}
3220
3221	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3222	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3223	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3224	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3225	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3226
3227	/* Enable Pause-Loop exiting. */
3228	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3229	&& pVM->hm.s.vmx.cPleGapTicks
3230	&& pVM->hm.s.vmx.cPleWindowTicks)
3231	{
3232	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3233
3234	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3235	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3236	AssertRCReturn(rc, rc);
3237	}
3238
3239	if ((fVal & fZap) != fVal)
3240	{
3241	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3242	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3243	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3244	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3245	}
3246
3247	/* Commit it to the VMCS and update our cache. */
3248	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3249	AssertRCReturn(rc, rc);
3250	pVmcsInfo->u32ProcCtls2 = fVal;
3251
3252	return VINF_SUCCESS;
3253	}
3254
3255
3256	/**
3257	* Sets up processor-based VM-execution controls in the VMCS.
3258	*
3259	* @returns VBox status code.
3260	* @param pVCpu The cross context virtual CPU structure.
3261	* @param pVmcsInfo The VMCS info. object.
3262	*/
3263	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3264	{
3265	PVM pVM = pVCpu->CTX_SUFF(pVM);
3266
3267	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3268	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3269
3270	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3271	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3272	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3273	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3274	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3275	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3276	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3277
3278	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3279	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3280	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3281	{
3282	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3283	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3284	}
3285
3286	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3287	if (!pVM->hm.s.fNestedPaging)
3288	{
3289	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3290	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3291	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3292	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3293	}
3294
3295	/* Use TPR shadowing if supported by the CPU. */
3296	if ( PDMHasApic(pVM)
3297	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3298	{
3299	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3300	/* CR8 writes cause a VM-exit based on TPR threshold. */
3301	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3302	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3303	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3304	AssertRCReturn(rc, rc);
3305	}
3306	else
3307	{
3308	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3309	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3310	if (pVM->hm.s.fAllow64BitGuests)
3311	{
3312	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3313	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3314	}
3315	}
3316
3317	/* Use MSR-bitmaps if supported by the CPU. */
3318	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3319	{
3320	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3321	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3322	AssertRCReturn(rc, rc);
3323	}
3324
3325	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3326	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3327	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3328
3329	if ((fVal & fZap) != fVal)
3330	{
3331	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3332	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3333	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3334	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3335	}
3336
3337	/* Commit it to the VMCS and update our cache. */
3338	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3339	AssertRCReturn(rc, rc);
3340	pVmcsInfo->u32ProcCtls = fVal;
3341
3342	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3343	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3344	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3345
3346	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3347	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3348	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3349
3350	/* Sanity check, should not really happen. */
3351	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3352	{ /* likely */ }
3353	else
3354	{
3355	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3356	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3357	}
3358
3359	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3360	return VINF_SUCCESS;
3361	}
3362
3363
3364	/**
3365	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3366	* Processor-based VM-execution) control fields in the VMCS.
3367	*
3368	* @returns VBox status code.
3369	* @param pVCpu The cross context virtual CPU structure.
3370	* @param pVmcsInfo The VMCS info. object.
3371	*/
3372	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3373	{
3374	/* Set the auto-load/store MSR area addresses in the VMCS. */
3375	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3376	if (RT_SUCCESS(rc))
3377	{
3378	/* Set the VMCS link pointer in the VMCS. */
3379	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3380	if (RT_SUCCESS(rc))
3381	{
3382	/* Set the CR0/CR4 guest/host mask. */
3383	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3384	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3385	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3386	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3387	if (RT_SUCCESS(rc))
3388	{
3389	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3390	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3391	return VINF_SUCCESS;
3392	}
3393	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3394	}
3395	else
3396	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3397	}
3398	else
3399	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3400	return rc;
3401	}
3402
3403
3404	/**
3405	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3406	*
3407	* We shall setup those exception intercepts that don't change during the
3408	* lifetime of the VM here. The rest are done dynamically while loading the
3409	* guest state.
3410	*
3411	* @returns VBox status code.
3412	* @param pVCpu The cross context virtual CPU structure.
3413	* @param pVmcsInfo The VMCS info. object.
3414	*/
3415	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3416	{
3417	/*
3418	* The following exceptions are always intercepted:
3419	*
3420	* #AC - To prevent the guest from hanging the CPU.
3421	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3422	* recursive #DBs can cause a CPU hang.
3423	* #PF - To sync our shadow page tables when nested-paging is not used.
3424	*/
3425	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3426	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3427	\| RT_BIT(X86_XCPT_DB)
3428	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3429
3430	/* Commit it to the VMCS. */
3431	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3432	AssertRCReturn(rc, rc);
3433
3434	/* Update our cache of the exception bitmap. */
3435	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3436	return VINF_SUCCESS;
3437	}
3438
3439
3440	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3441	/**
3442	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3443	*
3444	* @returns VBox status code.
3445	* @param pVCpu The cross context virtual CPU structure.
3446	* @param pVmcsInfo The VMCS info. object.
3447	*/
3448	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3449	{
3450	PVM pVM = pVCpu->CTX_SUFF(pVM);
3451	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3452	if (RT_SUCCESS(rc))
3453	{
3454	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3455	if (RT_SUCCESS(rc))
3456	{
3457	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3458	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3459	if (RT_SUCCESS(rc))
3460	{
3461	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3462	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3463	if (RT_SUCCESS(rc))
3464	return VINF_SUCCESS;
3465
3466	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3467	}
3468	else
3469	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3470	}
3471	else
3472	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3473	}
3474	else
3475	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3476
3477	return rc;
3478	}
3479	#endif
3480
3481
3482	/**
3483	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3484	* VMX.
3485	*
3486	* @returns VBox status code.
3487	* @param pVCpu The cross context virtual CPU structure.
3488	* @param pVmcsInfo The VMCS info. object.
3489	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3490	*/
3491	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3492	{
3493	Assert(pVmcsInfo->pvVmcs);
3494	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3495
3496	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3497	PVM pVM = pVCpu->CTX_SUFF(pVM);
3498	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3499	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3500
3501	LogFlowFunc(("\n"));
3502
3503	/*
3504	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3505	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3506	*/
3507	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3508	if (RT_SUCCESS(rc))
3509	{
3510	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3511	if (RT_SUCCESS(rc))
3512	{
3513	if (!fIsNstGstVmcs)
3514	{
3515	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3516	if (RT_SUCCESS(rc))
3517	{
3518	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3519	if (RT_SUCCESS(rc))
3520	{
3521	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3522	if (RT_SUCCESS(rc))
3523	{
3524	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3525	if (RT_SUCCESS(rc))
3526	{ /* likely */ }
3527	else
3528	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3529	}
3530	else
3531	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3532	}
3533	else
3534	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3535	}
3536	else
3537	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3538	}
3539	else
3540	{
3541	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3542	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3543	if (RT_SUCCESS(rc))
3544	{ /* likely */ }
3545	else
3546	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3547	#else
3548	AssertFailed();
3549	#endif
3550	}
3551	}
3552	else
3553	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3554	}
3555	else
3556	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3557
3558	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3559	if (RT_SUCCESS(rc))
3560	{
3561	rc = hmR0VmxClearVmcs(pVmcsInfo);
3562	if (RT_SUCCESS(rc))
3563	{ /* likely */ }
3564	else
3565	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3566	}
3567
3568	/*
3569	* Update the last-error record both for failures and success, so we
3570	* can propagate the status code back to ring-3 for diagnostics.
3571	*/
3572	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3573	NOREF(pszVmcs);
3574	return rc;
3575	}
3576
3577
3578	/**
3579	* Does global VT-x initialization (called during module initialization).
3580	*
3581	* @returns VBox status code.
3582	*/
3583	VMMR0DECL(int) VMXR0GlobalInit(void)
3584	{
3585	#ifdef HMVMX_USE_FUNCTION_TABLE
3586	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3587	# ifdef VBOX_STRICT
3588	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3589	Assert(g_apfnVMExitHandlers[i]);
3590	# endif
3591	#endif
3592	return VINF_SUCCESS;
3593	}
3594
3595
3596	/**
3597	* Does global VT-x termination (called during module termination).
3598	*/
3599	VMMR0DECL(void) VMXR0GlobalTerm()
3600	{
3601	/* Nothing to do currently. */
3602	}
3603
3604
3605	/**
3606	* Sets up and activates VT-x on the current CPU.
3607	*
3608	* @returns VBox status code.
3609	* @param pHostCpu The HM physical-CPU structure.
3610	* @param pVM The cross context VM structure. Can be
3611	* NULL after a host resume operation.
3612	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3613	* fEnabledByHost is @c true).
3614	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3615	* @a fEnabledByHost is @c true).
3616	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3617	* enable VT-x on the host.
3618	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3619	*/
3620	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3621	PCSUPHWVIRTMSRS pHwvirtMsrs)
3622	{
3623	AssertPtr(pHostCpu);
3624	AssertPtr(pHwvirtMsrs);
3625	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3626
3627	/* Enable VT-x if it's not already enabled by the host. */
3628	if (!fEnabledByHost)
3629	{
3630	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3631	if (RT_FAILURE(rc))
3632	return rc;
3633	}
3634
3635	/*
3636	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3637	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3638	* invalidated when flushing by VPID.
3639	*/
3640	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3641	{
3642	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3643	pHostCpu->fFlushAsidBeforeUse = false;
3644	}
3645	else
3646	pHostCpu->fFlushAsidBeforeUse = true;
3647
3648	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3649	++pHostCpu->cTlbFlushes;
3650
3651	return VINF_SUCCESS;
3652	}
3653
3654
3655	/**
3656	* Deactivates VT-x on the current CPU.
3657	*
3658	* @returns VBox status code.
3659	* @param pvCpuPage Pointer to the VMXON region.
3660	* @param HCPhysCpuPage Physical address of the VMXON region.
3661	*
3662	* @remarks This function should never be called when SUPR0EnableVTx() or
3663	* similar was used to enable VT-x on the host.
3664	*/
3665	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3666	{
3667	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3668
3669	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3670	return hmR0VmxLeaveRootMode();
3671	}
3672
3673
3674	/**
3675	* Does per-VM VT-x initialization.
3676	*
3677	* @returns VBox status code.
3678	* @param pVM The cross context VM structure.
3679	*/
3680	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3681	{
3682	AssertPtr(pVM);
3683	LogFlowFunc(("pVM=%p\n", pVM));
3684
3685	int rc = hmR0VmxStructsAlloc(pVM);
3686	if (RT_FAILURE(rc))
3687	{
3688	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3689	return rc;
3690	}
3691
3692	return VINF_SUCCESS;
3693	}
3694
3695
3696	/**
3697	* Does per-VM VT-x termination.
3698	*
3699	* @returns VBox status code.
3700	* @param pVM The cross context VM structure.
3701	*/
3702	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3703	{
3704	AssertPtr(pVM);
3705	LogFlowFunc(("pVM=%p\n", pVM));
3706
3707	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3708	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3709	{
3710	Assert(pVM->hm.s.vmx.pvScratch);
3711	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3712	}
3713	#endif
3714	hmR0VmxStructsFree(pVM);
3715	return VINF_SUCCESS;
3716	}
3717
3718
3719	/**
3720	* Sets up the VM for execution using hardware-assisted VMX.
3721	* This function is only called once per-VM during initialization.
3722	*
3723	* @returns VBox status code.
3724	* @param pVM The cross context VM structure.
3725	*/
3726	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3727	{
3728	AssertPtr(pVM);
3729	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3730
3731	LogFlowFunc(("pVM=%p\n", pVM));
3732
3733	/*
3734	* At least verify if VMX is enabled, since we can't check if we're in
3735	* VMX root mode or not without causing a #GP.
3736	*/
3737	RTCCUINTREG const uHostCr4 = ASMGetCR4();
3738	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
3739	{ /* likely */ }
3740	else
3741	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3742
3743	/*
3744	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3745	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3746	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3747	*/
3748	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3749	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3750	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3751	{
3752	LogRelFunc(("Invalid real-on-v86 state.\n"));
3753	return VERR_INTERNAL_ERROR;
3754	}
3755
3756	/* Initialize these always, see hmR3InitFinalizeR0().*/
3757	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3758	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3759
3760	/* Setup the tagged-TLB flush handlers. */
3761	int rc = hmR0VmxSetupTaggedTlb(pVM);
3762	if (RT_FAILURE(rc))
3763	{
3764	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3765	return rc;
3766	}
3767
3768	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3769	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3770	#if HC_ARCH_BITS == 64
3771	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3772	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3773	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3774	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3775	#endif
3776
3777	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3778	{
3779	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3780	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3781
3782	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3783	if (RT_SUCCESS(rc))
3784	{
3785	#if HC_ARCH_BITS == 32
3786	hmR0VmxInitVmcsReadCache(pVCpu);
3787	#endif
3788	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3789	if (pVM->cpum.ro.GuestFeatures.fVmx)
3790	{
3791	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3792	if (RT_SUCCESS(rc))
3793	{ /* likely */ }
3794	else
3795	{
3796	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3797	return rc;
3798	}
3799	}
3800	#endif
3801	}
3802	else
3803	{
3804	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3805	return rc;
3806	}
3807	}
3808
3809	return VINF_SUCCESS;
3810	}
3811
3812
3813	#if HC_ARCH_BITS == 32
3814	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3815	/**
3816	* Check if guest state allows safe use of 32-bit switcher again.
3817	*
3818	* Segment bases and protected mode structures must be 32-bit addressable
3819	* because the 32-bit switcher will ignore high dword when writing these VMCS
3820	* fields. See @bugref{8432} for details.
3821	*
3822	* @returns true if safe, false if must continue to use the 64-bit switcher.
3823	* @param pCtx Pointer to the guest-CPU context.
3824	*
3825	* @remarks No-long-jump zone!!!
3826	*/
3827	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3828	{
3829	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3830	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3831	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3832	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3833	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3834	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3835	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3836	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3837	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3838	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3839
3840	/* All good, bases are 32-bit. */
3841	return true;
3842	}
3843	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3844
3845	# ifdef VBOX_STRICT
3846	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3847	{
3848	switch (idxField)
3849	{
3850	case VMX_VMCS_GUEST_RIP:
3851	case VMX_VMCS_GUEST_RSP:
3852	case VMX_VMCS_GUEST_SYSENTER_EIP:
3853	case VMX_VMCS_GUEST_SYSENTER_ESP:
3854	case VMX_VMCS_GUEST_GDTR_BASE:
3855	case VMX_VMCS_GUEST_IDTR_BASE:
3856	case VMX_VMCS_GUEST_CS_BASE:
3857	case VMX_VMCS_GUEST_DS_BASE:
3858	case VMX_VMCS_GUEST_ES_BASE:
3859	case VMX_VMCS_GUEST_FS_BASE:
3860	case VMX_VMCS_GUEST_GS_BASE:
3861	case VMX_VMCS_GUEST_SS_BASE:
3862	case VMX_VMCS_GUEST_LDTR_BASE:
3863	case VMX_VMCS_GUEST_TR_BASE:
3864	case VMX_VMCS_GUEST_CR3:
3865	return true;
3866	}
3867	return false;
3868	}
3869
3870	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3871	{
3872	switch (idxField)
3873	{
3874	/* Read-only fields. */
3875	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3876	return true;
3877	}
3878	/* Remaining readable fields should also be writable. */
3879	return hmR0VmxIsValidWriteField(idxField);
3880	}
3881	# endif /* VBOX_STRICT */
3882
3883
3884	/**
3885	* Executes the specified handler in 64-bit mode.
3886	*
3887	* @returns VBox status code (no informational status codes).
3888	* @param pVCpu The cross context virtual CPU structure.
3889	* @param enmOp The operation to perform.
3890	* @param cParams Number of parameters.
3891	* @param paParam Array of 32-bit parameters.
3892	*/
3893	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3894	{
3895	AssertPtr(pVCpu);
3896	PVM pVM = pVCpu->CTX_SUFF(pVM);
3897	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3898	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3899	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3900	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3901
3902	#ifdef VBOX_STRICT
3903	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3904	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3905
3906	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3907	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3908	#endif
3909
3910	/* Disable interrupts. */
3911	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3912
3913	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3914	RTCPUID idHostCpu = RTMpCpuId();
3915	CPUMR0SetLApic(pVCpu, idHostCpu);
3916	#endif
3917
3918	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3919
3920	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3921	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3922
3923	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3924	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3925	hmR0VmxClearVmcs(pVmcsInfo);
3926
3927	/* Leave VMX root mode and disable VMX. */
3928	VMXDisable();
3929	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3930
3931	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3932	CPUMSetHyperEIP(pVCpu, enmOp);
3933	for (int i = (int)cParams - 1; i >= 0; i--)
3934	CPUMPushHyper(pVCpu, paParam[i]);
3935
3936	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3937
3938	/* Call the switcher. */
3939	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3940	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3941
3942	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3943	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3944
3945	/* Re-enter VMX root mode. */
3946	int rc2 = VMXEnable(HCPhysCpuPage);
3947	if (RT_FAILURE(rc2))
3948	{
3949	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3950	ASMSetFlags(fOldEFlags);
3951	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3952	return rc2;
3953	}
3954
3955	/* Restore the VMCS as the current VMCS. */
3956	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3957	AssertRC(rc2);
3958	Assert(!(ASMGetFlags() & X86_EFL_IF));
3959	ASMSetFlags(fOldEFlags);
3960	return rc;
3961	}
3962
3963
3964	/**
3965	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3966	* supporting 64-bit guests.
3967	*
3968	* @returns VBox status code.
3969	* @param fResume Whether to VMLAUNCH or VMRESUME.
3970	* @param pCtx Pointer to the guest-CPU context.
3971	* @param pCache Pointer to the VMCS batch cache.
3972	* @param pVM The cross context VM structure.
3973	* @param pVCpu The cross context virtual CPU structure.
3974	*/
3975	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3976	{
3977	NOREF(fResume);
3978
3979	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3980	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3981	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3982
3983	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3984	pCache->uPos = 1;
3985	pCache->interPD = PGMGetInterPaeCR3(pVM);
3986	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3987	#endif
3988
3989	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3990	pCache->TestIn.HCPhysCpuPage = 0;
3991	pCache->TestIn.HCPhysVmcs = 0;
3992	pCache->TestIn.pCache = 0;
3993	pCache->TestOut.HCPhysVmcs = 0;
3994	pCache->TestOut.pCache = 0;
3995	pCache->TestOut.pCtx = 0;
3996	pCache->TestOut.eflags = 0;
3997	#else
3998	NOREF(pCache);
3999	#endif
4000
4001	uint32_t aParam[10];
4002	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4003	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4004	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4005	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4006	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4007	aParam[5] = 0;
4008	aParam[6] = VM_RC_ADDR(pVM, pVM);
4009	aParam[7] = 0;
4010	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4011	aParam[9] = 0;
4012
4013	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4014	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4015	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4016	#endif
4017	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4018
4019	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4020	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4021	Assert(pCtx->dr[4] == 10);
4022	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4023	#endif
4024
4025	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4026	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4027	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4028	pVmcsInfo->HCPhysVmcs));
4029	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4030	pCache->TestOut.HCPhysVmcs));
4031	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4032	pCache->TestOut.pCache));
4033	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4034	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4035	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4036	pCache->TestOut.pCtx));
4037	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4038	#endif
4039	NOREF(pCtx);
4040	return rc;
4041	}
4042	#endif
4043
4044
4045	/**
4046	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4047	* the VMCS.
4048	*
4049	* @returns VBox status code.
4050	*/
4051	static int hmR0VmxExportHostControlRegs(void)
4052	{
4053	RTCCUINTREG uReg = ASMGetCR0();
4054	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4055	AssertRCReturn(rc, rc);
4056
4057	uReg = ASMGetCR3();
4058	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4059	AssertRCReturn(rc, rc);
4060
4061	uReg = ASMGetCR4();
4062	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4063	AssertRCReturn(rc, rc);
4064	return rc;
4065	}
4066
4067
4068	/**
4069	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4070	* the host-state area in the VMCS.
4071	*
4072	* @returns VBox status code.
4073	* @param pVCpu The cross context virtual CPU structure.
4074	*/
4075	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4076	{
4077	#if HC_ARCH_BITS == 64
4078	/**
4079	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4080	* requirements. See hmR0VmxExportHostSegmentRegs().
4081	*/
4082	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4083	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4084	{ \
4085	bool fValidSelector = true; \
4086	if ((selValue) & X86_SEL_LDT) \
4087	{ \
4088	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4089	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4090	} \
4091	if (fValidSelector) \
4092	{ \
4093	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4094	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4095	} \
4096	(selValue) = 0; \
4097	}
4098
4099	/*
4100	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4101	* will be messed up. We should -not- save the messed up state without restoring
4102	* the original host-state, see @bugref{7240}.
4103	*
4104	* This apparently can happen (most likely the FPU changes), deal with it rather than
4105	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4106	*/
4107	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4108	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4109	{
4110	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4111	pVCpu->idCpu));
4112	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4113	}
4114	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4115	#else
4116	RT_NOREF(pVCpu);
4117	#endif
4118
4119	/*
4120	* Host DS, ES, FS and GS segment registers.
4121	*/
4122	#if HC_ARCH_BITS == 64
4123	RTSEL uSelDS = ASMGetDS();
4124	RTSEL uSelES = ASMGetES();
4125	RTSEL uSelFS = ASMGetFS();
4126	RTSEL uSelGS = ASMGetGS();
4127	#else
4128	RTSEL uSelDS = 0;
4129	RTSEL uSelES = 0;
4130	RTSEL uSelFS = 0;
4131	RTSEL uSelGS = 0;
4132	#endif
4133
4134	/*
4135	* Host CS and SS segment registers.
4136	*/
4137	RTSEL uSelCS = ASMGetCS();
4138	RTSEL uSelSS = ASMGetSS();
4139
4140	/*
4141	* Host TR segment register.
4142	*/
4143	RTSEL uSelTR = ASMGetTR();
4144
4145	#if HC_ARCH_BITS == 64
4146	/*
4147	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4148	* gain VM-entry and restore them before we get preempted.
4149	*
4150	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4151	*/
4152	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4153	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4154	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4155	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4156	# undef VMXLOCAL_ADJUST_HOST_SEG
4157	#endif
4158
4159	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4160	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4161	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4162	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4163	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4164	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4165	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4166	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4167	Assert(uSelCS);
4168	Assert(uSelTR);
4169
4170	/* Write these host selector fields into the host-state area in the VMCS. */
4171	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4172	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4173	#if HC_ARCH_BITS == 64
4174	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4175	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4176	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4177	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4178	#else
4179	NOREF(uSelDS);
4180	NOREF(uSelES);
4181	NOREF(uSelFS);
4182	NOREF(uSelGS);
4183	#endif
4184	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4185	AssertRCReturn(rc, rc);
4186
4187	/*
4188	* Host GDTR and IDTR.
4189	*/
4190	RTGDTR Gdtr;
4191	RTIDTR Idtr;
4192	RT_ZERO(Gdtr);
4193	RT_ZERO(Idtr);
4194	ASMGetGDTR(&Gdtr);
4195	ASMGetIDTR(&Idtr);
4196	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4197	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4198	AssertRCReturn(rc, rc);
4199
4200	#if HC_ARCH_BITS == 64
4201	/*
4202	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4203	* them to the maximum limit (0xffff) on every VM-exit.
4204	*/
4205	if (Gdtr.cbGdt != 0xffff)
4206	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4207
4208	/*
4209	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4210	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4211	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4212	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4213	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4214	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4215	* at 0xffff on hosts where we are sure it won't cause trouble.
4216	*/
4217	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4218	if (Idtr.cbIdt < 0x0fff)
4219	# else
4220	if (Idtr.cbIdt != 0xffff)
4221	# endif
4222	{
4223	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4224	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4225	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4226	}
4227	#endif
4228
4229	/*
4230	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4231	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4232	* RPL should be too in most cases.
4233	*/
4234	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4235	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4236
4237	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4238	#if HC_ARCH_BITS == 64
4239	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4240
4241	/*
4242	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4243	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4244	* restoration if the host has something else. Task switching is not supported in 64-bit
4245	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4246	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4247	*
4248	* [1] See Intel spec. 3.5 "System Descriptor Types".
4249	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4250	*/
4251	PVM pVM = pVCpu->CTX_SUFF(pVM);
4252	Assert(pDesc->System.u4Type == 11);
4253	if ( pDesc->System.u16LimitLow != 0x67
4254	\|\| pDesc->System.u4LimitHigh)
4255	{
4256	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4257	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4258	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4259	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4260	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4261	}
4262
4263	/*
4264	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4265	*/
4266	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4267	{
4268	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4269	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4270	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4271	{
4272	/* The GDT is read-only but the writable GDT is available. */
4273	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4274	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4275	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4276	AssertRCReturn(rc, rc);
4277	}
4278	}
4279	#else
4280	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4281	#endif
4282	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4283	AssertRCReturn(rc, rc);
4284
4285	/*
4286	* Host FS base and GS base.
4287	*/
4288	#if HC_ARCH_BITS == 64
4289	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4290	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4291	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4292	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4293	AssertRCReturn(rc, rc);
4294
4295	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4296	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4297	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4298	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4299	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4300	#endif
4301	return VINF_SUCCESS;
4302	}
4303
4304
4305	/**
4306	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4307	* host-state area of the VMCS.
4308	*
4309	* These MSRs will be automatically restored on the host after every successful
4310	* VM-exit.
4311	*
4312	* @returns VBox status code.
4313	* @param pVCpu The cross context virtual CPU structure.
4314	*
4315	* @remarks No-long-jump zone!!!
4316	*/
4317	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4318	{
4319	AssertPtr(pVCpu);
4320
4321	/*
4322	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4323	* rather than swapping them on every VM-entry.
4324	*/
4325	hmR0VmxLazySaveHostMsrs(pVCpu);
4326
4327	/*
4328	* Host Sysenter MSRs.
4329	*/
4330	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4331	#if HC_ARCH_BITS == 32
4332	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4333	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4334	#else
4335	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4336	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4337	#endif
4338	AssertRCReturn(rc, rc);
4339
4340	/*
4341	* Host EFER MSR.
4342	*
4343	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4344	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4345	*/
4346	PVM pVM = pVCpu->CTX_SUFF(pVM);
4347	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4348	{
4349	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4350	AssertRCReturn(rc, rc);
4351	}
4352
4353	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4354	* hmR0VmxExportGuestEntryExitCtls(). */
4355
4356	return VINF_SUCCESS;
4357	}
4358
4359
4360	/**
4361	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4362	*
4363	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4364	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4365	*
4366	* @returns true if we need to load guest EFER, false otherwise.
4367	* @param pVCpu The cross context virtual CPU structure.
4368	*
4369	* @remarks Requires EFER, CR4.
4370	* @remarks No-long-jump zone!!!
4371	*/
4372	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4373	{
4374	#ifdef HMVMX_ALWAYS_SWAP_EFER
4375	RT_NOREF(pVCpu);
4376	return true;
4377	#else
4378	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4379	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4380	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4381	if (CPUMIsGuestInLongModeEx(pCtx))
4382	return false;
4383	#endif
4384
4385	PVM pVM = pVCpu->CTX_SUFF(pVM);
4386	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4387	uint64_t const u64GuestEfer = pCtx->msrEFER;
4388
4389	/*
4390	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4391	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4392	*/
4393	if ( CPUMIsGuestInLongModeEx(pCtx)
4394	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4395	return true;
4396
4397	/*
4398	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4399	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4400	*
4401	* See Intel spec. 4.5 "IA-32e Paging".
4402	* See Intel spec. 4.1.1 "Three Paging Modes".
4403	*
4404	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4405	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4406	*/
4407	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4408	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4409	if ( (pCtx->cr4 & X86_CR4_PAE)
4410	&& (pCtx->cr0 & X86_CR0_PG)
4411	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4412	{
4413	/* Assert that host is NX capable. */
4414	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4415	return true;
4416	}
4417
4418	return false;
4419	#endif
4420	}
4421
4422	/**
4423	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4424	* VMCS.
4425	*
4426	* This is typically required when the guest changes paging mode.
4427	*
4428	* @returns VBox status code.
4429	* @param pVCpu The cross context virtual CPU structure.
4430	* @param pVmxTransient The VMX-transient structure.
4431	*
4432	* @remarks Requires EFER.
4433	* @remarks No-long-jump zone!!!
4434	*/
4435	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4436	{
4437	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4438	{
4439	PVM pVM = pVCpu->CTX_SUFF(pVM);
4440	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4441
4442	/*
4443	* VM-entry controls.
4444	*/
4445	{
4446	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4447	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4448
4449	/*
4450	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4451	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4452	*
4453	* For nested-guests, this is a mandatory VM-entry control. It's also
4454	* required because we do not want to leak host bits to the nested-guest.
4455	*/
4456	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4457
4458	/*
4459	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4460	*
4461	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4462	* required to get the nested-guest working with hardware-assisted VMX execution.
4463	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4464	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4465	* here rather than while merging the guest VMCS controls.
4466	*/
4467	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4468	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4469	else
4470	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4471
4472	/*
4473	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4474	*
4475	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4476	* regardless of whether the nested-guest VMCS specifies it because we are free to
4477	* load whatever MSRs we require and we do not need to modify the guest visible copy
4478	* of the VM-entry MSR load area.
4479	*/
4480	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4481	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4482	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4483	else
4484	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4485
4486	/*
4487	* The following should -not- be set (since we're not in SMM mode):
4488	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4489	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4490	*/
4491
4492	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4493	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4494
4495	if ((fVal & fZap) == fVal)
4496	{ /* likely */ }
4497	else
4498	{
4499	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4500	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4501	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4502	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4503	}
4504
4505	/* Commit it to the VMCS. */
4506	if (pVmcsInfo->u32EntryCtls != fVal)
4507	{
4508	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4509	AssertRCReturn(rc, rc);
4510	pVmcsInfo->u32EntryCtls = fVal;
4511	}
4512	}
4513
4514	/*
4515	* VM-exit controls.
4516	*/
4517	{
4518	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4519	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4520
4521	/*
4522	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4523	* supported the 1-setting of this bit.
4524	*
4525	* For nested-guests, we set the "save debug controls" as the converse
4526	* "load debug controls" is mandatory for nested-guests anyway.
4527	*/
4528	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4529
4530	/*
4531	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4532	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4533	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4534	* hmR0VmxExportHostMsrs().
4535	*
4536	* For nested-guests, we always set this bit as we do not support 32-bit
4537	* hosts.
4538	*/
4539	#if HC_ARCH_BITS == 64
4540	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4541	#else
4542	Assert(!pVmxTransient->fIsNestedGuest);
4543	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4544	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4545	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4546	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4547	{
4548	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4549	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4550	}
4551	else
4552	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4553	#endif
4554
4555	/*
4556	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4557	*
4558	* For nested-guests, we should use the "save IA32_EFER" control if we also
4559	* used the "load IA32_EFER" control while exporting VM-entry controls.
4560	*/
4561	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4562	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4563	{
4564	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4565	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4566	}
4567
4568	/*
4569	* Enable saving of the VMX-preemption timer value on VM-exit.
4570	* For nested-guests, currently not exposed/used.
4571	*/
4572	if ( pVM->hm.s.vmx.fUsePreemptTimer
4573	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4574	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4575
4576	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4577	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4578
4579	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4580	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4581	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4582
4583	if ((fVal & fZap) == fVal)
4584	{ /* likely */ }
4585	else
4586	{
4587	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4588	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4589	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4590	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4591	}
4592
4593	/* Commit it to the VMCS. */
4594	if (pVmcsInfo->u32ExitCtls != fVal)
4595	{
4596	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4597	AssertRCReturn(rc, rc);
4598	pVmcsInfo->u32ExitCtls = fVal;
4599	}
4600	}
4601
4602	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4603	}
4604	return VINF_SUCCESS;
4605	}
4606
4607
4608	/**
4609	* Sets the TPR threshold in the VMCS.
4610	*
4611	* @returns VBox status code.
4612	* @param pVCpu The cross context virtual CPU structure.
4613	* @param pVmcsInfo The VMCS info. object.
4614	* @param u32TprThreshold The TPR threshold (task-priority class only).
4615	*/
4616	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4617	{
4618	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4619	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4620	RT_NOREF2(pVCpu, pVmcsInfo);
4621	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4622	}
4623
4624
4625	/**
4626	* Exports the guest APIC TPR state into the VMCS.
4627	*
4628	* @returns VBox status code.
4629	* @param pVCpu The cross context virtual CPU structure.
4630	* @param pVmxTransient The VMX-transient structure.
4631	*
4632	* @remarks No-long-jump zone!!!
4633	*/
4634	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4635	{
4636	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4637	{
4638	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4639
4640	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4641	if (!pVmxTransient->fIsNestedGuest)
4642	{
4643	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4644	&& APICIsEnabled(pVCpu))
4645	{
4646	/*
4647	* Setup TPR shadowing.
4648	*/
4649	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4650	{
4651	bool fPendingIntr = false;
4652	uint8_t u8Tpr = 0;
4653	uint8_t u8PendingIntr = 0;
4654	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4655	AssertRCReturn(rc, rc);
4656
4657	/*
4658	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4659	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4660	* priority of the pending interrupt so we can deliver the interrupt. If there
4661	* are no interrupts pending, set threshold to 0 to not cause any
4662	* TPR-below-threshold VM-exits.
4663	*/
4664	Assert(pVmcsInfo->pbVirtApic);
4665	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4666	uint32_t u32TprThreshold = 0;
4667	if (fPendingIntr)
4668	{
4669	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4670	(which is the Task-Priority Class). */
4671	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4672	const uint8_t u8TprPriority = u8Tpr >> 4;
4673	if (u8PendingPriority <= u8TprPriority)
4674	u32TprThreshold = u8PendingPriority;
4675	}
4676
4677	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4678	AssertRCReturn(rc, rc);
4679	}
4680	}
4681	}
4682	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4683	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4684	}
4685	return VINF_SUCCESS;
4686	}
4687
4688
4689	/**
4690	* Gets the guest interruptibility-state.
4691	*
4692	* @returns Guest's interruptibility-state.
4693	* @param pVCpu The cross context virtual CPU structure.
4694	* @param pVmxTransient The VMX-transient structure.
4695	*
4696	* @remarks No-long-jump zone!!!
4697	*/
4698	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4699	{
4700	/*
4701	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4702	*/
4703	uint32_t fIntrState = 0;
4704	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4705	{
4706	/* If inhibition is active, RIP and RFLAGS should've been updated
4707	(i.e. read previously from the VMCS or from ring-3). */
4708	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4709	#ifdef VBOX_STRICT
4710	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4711	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4712	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4713	#endif
4714	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4715	{
4716	if (pCtx->eflags.Bits.u1IF)
4717	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4718	else
4719	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4720	}
4721	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4722	{
4723	/*
4724	* We can clear the inhibit force flag as even if we go back to the recompiler
4725	* without executing guest code in VT-x, the flag's condition to be cleared is
4726	* met and thus the cleared state is correct.
4727	*/
4728	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4729	}
4730	}
4731
4732	/*
4733	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4734	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4735	* setting this would block host-NMIs and IRET will not clear the blocking.
4736	*
4737	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4738	*
4739	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4740	*/
4741	if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
4742	&& CPUMIsGuestNmiBlocking(pVCpu))
4743	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4744
4745	return fIntrState;
4746	}
4747
4748
4749	/**
4750	* Exports the exception intercepts required for guest execution in the VMCS.
4751	*
4752	* @returns VBox status code.
4753	* @param pVCpu The cross context virtual CPU structure.
4754	* @param pVmxTransient The VMX-transient structure.
4755	*
4756	* @remarks No-long-jump zone!!!
4757	*/
4758	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4759	{
4760	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4761	{
4762	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4763	if ( !pVmxTransient->fIsNestedGuest
4764	&& pVCpu->hm.s.fGIMTrapXcptUD)
4765	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4766	else
4767	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4768
4769	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4770	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4771	}
4772	return VINF_SUCCESS;
4773	}
4774
4775
4776	/**
4777	* Exports the guest's RIP into the guest-state area in the VMCS.
4778	*
4779	* @returns VBox status code.
4780	* @param pVCpu The cross context virtual CPU structure.
4781	*
4782	* @remarks No-long-jump zone!!!
4783	*/
4784	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4785	{
4786	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4787	{
4788	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4789
4790	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4791	AssertRCReturn(rc, rc);
4792
4793	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4794	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4795	}
4796	return VINF_SUCCESS;
4797	}
4798
4799
4800	/**
4801	* Exports the guest's RSP into the guest-state area in the VMCS.
4802	*
4803	* @returns VBox status code.
4804	* @param pVCpu The cross context virtual CPU structure.
4805	*
4806	* @remarks No-long-jump zone!!!
4807	*/
4808	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4809	{
4810	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4811	{
4812	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4813
4814	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4815	AssertRCReturn(rc, rc);
4816
4817	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4818	}
4819	return VINF_SUCCESS;
4820	}
4821
4822
4823	/**
4824	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4825	*
4826	* @returns VBox status code.
4827	* @param pVCpu The cross context virtual CPU structure.
4828	* @param pVmxTransient The VMX-transient structure.
4829	*
4830	* @remarks No-long-jump zone!!!
4831	*/
4832	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4833	{
4834	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4835	{
4836	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4837
4838	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4839	Let us assert it as such and use 32-bit VMWRITE. */
4840	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4841	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4842	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4843	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4844
4845	/*
4846	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4847	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4848	* can run the real-mode guest code under Virtual 8086 mode.
4849	*/
4850	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4851	if (pVmcsInfo->RealMode.fRealOnV86Active)
4852	{
4853	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4854	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4855	Assert(!pVmxTransient->fIsNestedGuest);
4856	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4857	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4858	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4859	}
4860
4861	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4862	AssertRCReturn(rc, rc);
4863
4864	/*
4865	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4866	*
4867	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4868	* through the hypervisor debugger using EFLAGS.TF.
4869	*/
4870	if ( !pVmxTransient->fIsNestedGuest
4871	&& !pVCpu->hm.s.fSingleInstruction
4872	&& fEFlags.Bits.u1TF)
4873	{
4874	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4875	* premature trips to ring-3 esp since IEM does not yet handle it. */
4876	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4877	AssertRCReturn(rc, rc);
4878	}
4879	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4880	* nested-guest VMCS. */
4881
4882	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4883	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4884	}
4885	return VINF_SUCCESS;
4886	}
4887
4888
4889	/**
4890	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4891	*
4892	* The guest FPU state is always pre-loaded hence we don't need to bother about
4893	* sharing FPU related CR0 bits between the guest and host.
4894	*
4895	* @returns VBox status code.
4896	* @param pVCpu The cross context virtual CPU structure.
4897	* @param pVmxTransient The VMX-transient structure.
4898	*
4899	* @remarks No-long-jump zone!!!
4900	*/
4901	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4902	{
4903	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4904	{
4905	PVM pVM = pVCpu->CTX_SUFF(pVM);
4906	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4907
4908	/*
4909	* Figure out fixed CR0 bits in VMX operation.
4910	*/
4911	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
4912	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4913	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4914	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4915	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4916	else
4917	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4918
4919	if (!pVmxTransient->fIsNestedGuest)
4920	{
4921	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4922	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4923	uint64_t const u64ShadowCr0 = u64GuestCr0;
4924	Assert(!RT_HI_U32(u64GuestCr0));
4925
4926	/*
4927	* Setup VT-x's view of the guest CR0.
4928	*/
4929	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4930	if (pVM->hm.s.fNestedPaging)
4931	{
4932	if (CPUMIsGuestPagingEnabled(pVCpu))
4933	{
4934	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4935	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4936	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4937	}
4938	else
4939	{
4940	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4941	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4942	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4943	}
4944
4945	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4946	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4947	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4948	}
4949	else
4950	{
4951	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4952	u64GuestCr0 \|= X86_CR0_WP;
4953	}
4954
4955	/*
4956	* Guest FPU bits.
4957	*
4958	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4959	* using CR0.TS.
4960	*
4961	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4962	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4963	*/
4964	u64GuestCr0 \|= X86_CR0_NE;
4965
4966	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4967	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4968
4969	/*
4970	* Update exception intercepts.
4971	*/
4972	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4973	if (pVmcsInfo->RealMode.fRealOnV86Active)
4974	{
4975	Assert(PDMVmmDevHeapIsEnabled(pVM));
4976	Assert(pVM->hm.s.vmx.pRealModeTSS);
4977	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4978	}
4979	else
4980	{
4981	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4982	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4983	if (fInterceptMF)
4984	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4985	}
4986
4987	/* Additional intercepts for debugging, define these yourself explicitly. */
4988	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4989	uXcptBitmap \|= 0
4990	\| RT_BIT(X86_XCPT_BP)
4991	\| RT_BIT(X86_XCPT_DE)
4992	\| RT_BIT(X86_XCPT_NM)
4993	\| RT_BIT(X86_XCPT_TS)
4994	\| RT_BIT(X86_XCPT_UD)
4995	\| RT_BIT(X86_XCPT_NP)
4996	\| RT_BIT(X86_XCPT_SS)
4997	\| RT_BIT(X86_XCPT_GP)
4998	\| RT_BIT(X86_XCPT_PF)
4999	\| RT_BIT(X86_XCPT_MF)
5000	;
5001	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5002	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5003	#endif
5004	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5005	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5006	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5007
5008	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5009	u64GuestCr0 \|= fSetCr0;
5010	u64GuestCr0 &= fZapCr0;
5011	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5012
5013	/* Commit the CR0 and related fields to the guest VMCS. */
5014	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5015	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5016	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5017	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5018	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5019	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5020	AssertRCReturn(rc, rc);
5021
5022	/* Update our caches. */
5023	pVmcsInfo->u32ProcCtls = uProcCtls;
5024	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5025
5026	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5027	}
5028	else
5029	{
5030	/*
5031	* With nested-guests, we may have extended the guest/host mask here since we
5032	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5033	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5034	* originally supplied. We must copy those bits from the nested-guest CR0 into
5035	* the nested-guest CR0 read-shadow.
5036	*/
5037	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5038	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5039	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5040	Assert(!RT_HI_U32(u64GuestCr0));
5041	Assert(u64GuestCr0 & X86_CR0_NE);
5042
5043	/*
5044	* Apply the hardware specified fixed CR0 bits and enable caching.
5045	* Note! We could be altering our VMX emulation's fixed bits. We thus
5046	* need to re-apply them while importing CR0.
5047	*/
5048	u64GuestCr0 \|= fSetCr0;
5049	u64GuestCr0 &= fZapCr0;
5050	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5051
5052	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5053	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5054	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5055	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5056	AssertRCReturn(rc, rc);
5057
5058	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5059	}
5060
5061	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5062	}
5063
5064	return VINF_SUCCESS;
5065	}
5066
5067
5068	/**
5069	* Exports the guest control registers (CR3, CR4) into the guest-state area
5070	* in the VMCS.
5071	*
5072	* @returns VBox strict status code.
5073	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5074	* without unrestricted guest access and the VMMDev is not presently
5075	* mapped (e.g. EFI32).
5076	*
5077	* @param pVCpu The cross context virtual CPU structure.
5078	* @param pVmxTransient The VMX-transient structure.
5079	*
5080	* @remarks No-long-jump zone!!!
5081	*/
5082	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5083	{
5084	int rc = VINF_SUCCESS;
5085	PVM pVM = pVCpu->CTX_SUFF(pVM);
5086
5087	/*
5088	* Guest CR2.
5089	* It's always loaded in the assembler code. Nothing to do here.
5090	*/
5091
5092	/*
5093	* Guest CR3.
5094	*/
5095	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5096	{
5097	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5098
5099	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5100	if (pVM->hm.s.fNestedPaging)
5101	{
5102	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5103	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5104
5105	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5106	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5107	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5108	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5109
5110	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5111	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5112	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5113
5114	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5115	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5116	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5117	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5118	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5119	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5120	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5121
5122	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5123	AssertRCReturn(rc, rc);
5124
5125	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5126	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5127	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5128	{
5129	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5130	if (CPUMIsGuestInPAEModeEx(pCtx))
5131	{
5132	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5133	AssertRCReturn(rc, rc);
5134	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5135	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5136	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5137	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5138	AssertRCReturn(rc, rc);
5139	}
5140
5141	/*
5142	* The guest's view of its CR3 is unblemished with nested paging when the
5143	* guest is using paging or we have unrestricted guest execution to handle
5144	* the guest when it's not using paging.
5145	*/
5146	GCPhysGuestCr3 = pCtx->cr3;
5147	}
5148	else
5149	{
5150	/*
5151	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5152	* thinks it accesses physical memory directly, we use our identity-mapped
5153	* page table to map guest-linear to guest-physical addresses. EPT takes care
5154	* of translating it to host-physical addresses.
5155	*/
5156	RTGCPHYS GCPhys;
5157	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5158
5159	/* We obtain it here every time as the guest could have relocated this PCI region. */
5160	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5161	if (RT_SUCCESS(rc))
5162	{ /* likely */ }
5163	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5164	{
5165	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5166	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5167	}
5168	else
5169	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5170
5171	GCPhysGuestCr3 = GCPhys;
5172	}
5173
5174	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5175	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5176	AssertRCReturn(rc, rc);
5177	}
5178	else
5179	{
5180	/* Non-nested paging case, just use the hypervisor's CR3. */
5181	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5182
5183	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5184	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5185	AssertRCReturn(rc, rc);
5186	}
5187
5188	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5189	}
5190
5191	/*
5192	* Guest CR4.
5193	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5194	*/
5195	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5196	{
5197	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5198	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5199
5200	/*
5201	* Figure out fixed CR4 bits in VMX operation.
5202	*/
5203	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5204	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5205	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5206
5207	/*
5208	* With nested-guests, we may have extended the guest/host mask here (since we
5209	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5210	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5211	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5212	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5213	*/
5214	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5215	uint64_t u64GuestCr4 = pCtx->cr4;
5216	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5217	? pCtx->cr4
5218	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5219	Assert(!RT_HI_U32(u64GuestCr4));
5220
5221	/*
5222	* Setup VT-x's view of the guest CR4.
5223	*
5224	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5225	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5226	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5227	*
5228	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5229	*/
5230	if (pVmcsInfo->RealMode.fRealOnV86Active)
5231	{
5232	Assert(pVM->hm.s.vmx.pRealModeTSS);
5233	Assert(PDMVmmDevHeapIsEnabled(pVM));
5234	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5235	}
5236
5237	if (pVM->hm.s.fNestedPaging)
5238	{
5239	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5240	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5241	{
5242	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5243	u64GuestCr4 \|= X86_CR4_PSE;
5244	/* Our identity mapping is a 32-bit page directory. */
5245	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5246	}
5247	/* else use guest CR4.*/
5248	}
5249	else
5250	{
5251	Assert(!pVmxTransient->fIsNestedGuest);
5252
5253	/*
5254	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5255	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5256	*/
5257	switch (pVCpu->hm.s.enmShadowMode)
5258	{
5259	case PGMMODE_REAL: /* Real-mode. */
5260	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5261	case PGMMODE_32_BIT: /* 32-bit paging. */
5262	{
5263	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5264	break;
5265	}
5266
5267	case PGMMODE_PAE: /* PAE paging. */
5268	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5269	{
5270	u64GuestCr4 \|= X86_CR4_PAE;
5271	break;
5272	}
5273
5274	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5275	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5276	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5277	break;
5278	#endif
5279	default:
5280	AssertFailed();
5281	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5282	}
5283	}
5284
5285	/*
5286	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
5287	* Note! For nested-guests, we could be altering our VMX emulation's
5288	* fixed bits. We thus need to re-apply them while importing CR4.
5289	*/
5290	u64GuestCr4 \|= fSetCr4;
5291	u64GuestCr4 &= fZapCr4;
5292
5293	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5294	/** @todo Fix to 64-bit when we drop 32-bit. */
5295	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5296	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5297	AssertRCReturn(rc, rc);
5298
5299	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5300	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5301
5302	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5303
5304	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5305	}
5306	return rc;
5307	}
5308
5309
5310	/**
5311	* Exports the guest debug registers into the guest-state area in the VMCS.
5312	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5313	*
5314	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5315	*
5316	* @returns VBox status code.
5317	* @param pVCpu The cross context virtual CPU structure.
5318	* @param pVmxTransient The VMX-transient structure.
5319	*
5320	* @remarks No-long-jump zone!!!
5321	*/
5322	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5323	{
5324	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5325
5326	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5327	* stepping. */
5328	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5329	if (pVmxTransient->fIsNestedGuest)
5330	{
5331	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5332	AssertRCReturn(rc, rc);
5333	return VINF_SUCCESS;
5334	}
5335
5336	#ifdef VBOX_STRICT
5337	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5338	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5339	{
5340	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5341	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5342	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5343	}
5344	#endif
5345
5346	bool fSteppingDB = false;
5347	bool fInterceptMovDRx = false;
5348	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5349	if (pVCpu->hm.s.fSingleInstruction)
5350	{
5351	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5352	PVM pVM = pVCpu->CTX_SUFF(pVM);
5353	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5354	{
5355	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5356	Assert(fSteppingDB == false);
5357	}
5358	else
5359	{
5360	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5361	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5362	pVCpu->hm.s.fClearTrapFlag = true;
5363	fSteppingDB = true;
5364	}
5365	}
5366
5367	uint32_t u32GuestDr7;
5368	if ( fSteppingDB
5369	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5370	{
5371	/*
5372	* Use the combined guest and host DRx values found in the hypervisor register set
5373	* because the hypervisor debugger has breakpoints active or someone is single stepping
5374	* on the host side without a monitor trap flag.
5375	*
5376	* Note! DBGF expects a clean DR6 state before executing guest code.
5377	*/
5378	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5379	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5380	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5381	{
5382	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5383	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5384	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5385	}
5386	else
5387	#endif
5388	if (!CPUMIsHyperDebugStateActive(pVCpu))
5389	{
5390	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5391	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5392	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5393	}
5394
5395	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5396	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5397	pVCpu->hm.s.fUsingHyperDR7 = true;
5398	fInterceptMovDRx = true;
5399	}
5400	else
5401	{
5402	/*
5403	* If the guest has enabled debug registers, we need to load them prior to
5404	* executing guest code so they'll trigger at the right time.
5405	*/
5406	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5407	{
5408	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5409	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5410	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5411	{
5412	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5413	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5414	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5415	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5416	}
5417	else
5418	#endif
5419	if (!CPUMIsGuestDebugStateActive(pVCpu))
5420	{
5421	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5422	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5423	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5424	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5425	}
5426	Assert(!fInterceptMovDRx);
5427	}
5428	/*
5429	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5430	* must intercept #DB in order to maintain a correct DR6 guest value, and
5431	* because we need to intercept it to prevent nested #DBs from hanging the
5432	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5433	*/
5434	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5435	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5436	&& !CPUMIsGuestDebugStateActive(pVCpu))
5437	#else
5438	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5439	#endif
5440	{
5441	fInterceptMovDRx = true;
5442	}
5443
5444	/* Update DR7 with the actual guest value. */
5445	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5446	pVCpu->hm.s.fUsingHyperDR7 = false;
5447	}
5448
5449	if (fInterceptMovDRx)
5450	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5451	else
5452	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5453
5454	/*
5455	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5456	* monitor-trap flag and update our cache.
5457	*/
5458	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5459	{
5460	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5461	AssertRCReturn(rc2, rc2);
5462	pVmcsInfo->u32ProcCtls = uProcCtls;
5463	}
5464
5465	/*
5466	* Update guest DR7.
5467	*/
5468	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5469	AssertRCReturn(rc, rc);
5470
5471	/*
5472	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5473	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5474	*
5475	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5476	*/
5477	if (fSteppingDB)
5478	{
5479	Assert(pVCpu->hm.s.fSingleInstruction);
5480	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5481
5482	uint32_t fIntrState = 0;
5483	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5484	AssertRCReturn(rc, rc);
5485
5486	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5487	{
5488	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5489	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5490	AssertRCReturn(rc, rc);
5491	}
5492	}
5493
5494	return VINF_SUCCESS;
5495	}
5496
5497
5498	#ifdef VBOX_STRICT
5499	/**
5500	* Strict function to validate segment registers.
5501	*
5502	* @param pVCpu The cross context virtual CPU structure.
5503	* @param pVmcsInfo The VMCS info. object.
5504	*
5505	* @remarks Will import guest CR0 on strict builds during validation of
5506	* segments.
5507	*/
5508	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5509	{
5510	/*
5511	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5512	*
5513	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5514	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5515	* unusable bit and doesn't change the guest-context value.
5516	*/
5517	PVM pVM = pVCpu->CTX_SUFF(pVM);
5518	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5519	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5520	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5521	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5522	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5523	{
5524	/* Protected mode checks */
5525	/* CS */
5526	Assert(pCtx->cs.Attr.n.u1Present);
5527	Assert(!(pCtx->cs.Attr.u & 0xf00));
5528	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5529	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5530	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5531	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5532	\|\| (pCtx->cs.Attr.n.u1Granularity));
5533	/* CS cannot be loaded with NULL in protected mode. */
5534	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5535	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5536	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5537	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5538	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5539	else
5540	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5541	/* SS */
5542	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5543	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5544	if ( !(pCtx->cr0 & X86_CR0_PE)
5545	\|\| pCtx->cs.Attr.n.u4Type == 3)
5546	{
5547	Assert(!pCtx->ss.Attr.n.u2Dpl);
5548	}
5549	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5550	{
5551	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5552	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5553	Assert(pCtx->ss.Attr.n.u1Present);
5554	Assert(!(pCtx->ss.Attr.u & 0xf00));
5555	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5556	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5557	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5558	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5559	\|\| (pCtx->ss.Attr.n.u1Granularity));
5560	}
5561	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5562	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5563	{
5564	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5565	Assert(pCtx->ds.Attr.n.u1Present);
5566	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5567	Assert(!(pCtx->ds.Attr.u & 0xf00));
5568	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5569	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5570	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5571	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5572	\|\| (pCtx->ds.Attr.n.u1Granularity));
5573	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5574	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5575	}
5576	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5577	{
5578	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5579	Assert(pCtx->es.Attr.n.u1Present);
5580	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5581	Assert(!(pCtx->es.Attr.u & 0xf00));
5582	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5583	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5584	\|\| !(pCtx->es.Attr.n.u1Granularity));
5585	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5586	\|\| (pCtx->es.Attr.n.u1Granularity));
5587	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5588	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5589	}
5590	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5591	{
5592	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5593	Assert(pCtx->fs.Attr.n.u1Present);
5594	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5595	Assert(!(pCtx->fs.Attr.u & 0xf00));
5596	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5597	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5598	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5599	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5600	\|\| (pCtx->fs.Attr.n.u1Granularity));
5601	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5602	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5603	}
5604	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5605	{
5606	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5607	Assert(pCtx->gs.Attr.n.u1Present);
5608	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5609	Assert(!(pCtx->gs.Attr.u & 0xf00));
5610	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5611	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5612	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5613	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5614	\|\| (pCtx->gs.Attr.n.u1Granularity));
5615	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5616	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5617	}
5618	/* 64-bit capable CPUs. */
5619	# if HC_ARCH_BITS == 64
5620	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5621	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5622	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5623	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5624	# endif
5625	}
5626	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5627	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5628	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5629	{
5630	/* Real and v86 mode checks. */
5631	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5632	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5633	if (pVmcsInfo->RealMode.fRealOnV86Active)
5634	{
5635	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5636	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5637	}
5638	else
5639	{
5640	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5641	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5642	}
5643
5644	/* CS */
5645	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5646	Assert(pCtx->cs.u32Limit == 0xffff);
5647	Assert(u32CSAttr == 0xf3);
5648	/* SS */
5649	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5650	Assert(pCtx->ss.u32Limit == 0xffff);
5651	Assert(u32SSAttr == 0xf3);
5652	/* DS */
5653	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5654	Assert(pCtx->ds.u32Limit == 0xffff);
5655	Assert(u32DSAttr == 0xf3);
5656	/* ES */
5657	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5658	Assert(pCtx->es.u32Limit == 0xffff);
5659	Assert(u32ESAttr == 0xf3);
5660	/* FS */
5661	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5662	Assert(pCtx->fs.u32Limit == 0xffff);
5663	Assert(u32FSAttr == 0xf3);
5664	/* GS */
5665	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5666	Assert(pCtx->gs.u32Limit == 0xffff);
5667	Assert(u32GSAttr == 0xf3);
5668	/* 64-bit capable CPUs. */
5669	# if HC_ARCH_BITS == 64
5670	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5671	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5672	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5673	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5674	# endif
5675	}
5676	}
5677	#endif /* VBOX_STRICT */
5678
5679
5680	/**
5681	* Exports a guest segment register into the guest-state area in the VMCS.
5682	*
5683	* @returns VBox status code.
5684	* @param pVCpu The cross context virtual CPU structure.
5685	* @param pVmcsInfo The VMCS info. object.
5686	* @param iSegReg The segment register number (X86_SREG_XXX).
5687	* @param pSelReg Pointer to the segment selector.
5688	*
5689	* @remarks No-long-jump zone!!!
5690	*/
5691	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5692	{
5693	Assert(iSegReg < X86_SREG_COUNT);
5694	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5695	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5696	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5697	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5698
5699	uint32_t u32Access = pSelReg->Attr.u;
5700	if (pVmcsInfo->RealMode.fRealOnV86Active)
5701	{
5702	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5703	u32Access = 0xf3;
5704	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5705	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5706	RT_NOREF_PV(pVCpu);
5707	}
5708	else
5709	{
5710	/*
5711	* The way to differentiate between whether this is really a null selector or was just
5712	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5713	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5714	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5715	* NULL selectors loaded in protected-mode have their attribute as 0.
5716	*/
5717	if (!u32Access)
5718	u32Access = X86DESCATTR_UNUSABLE;
5719	}
5720
5721	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5722	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5723	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5724
5725	/*
5726	* Commit it to the VMCS.
5727	*/
5728	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5729	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5730	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5731	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5732	AssertRCReturn(rc, rc);
5733	return rc;
5734	}
5735
5736
5737	/**
5738	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5739	* area in the VMCS.
5740	*
5741	* @returns VBox status code.
5742	* @param pVCpu The cross context virtual CPU structure.
5743	* @param pVmxTransient The VMX-transient structure.
5744	*
5745	* @remarks Will import guest CR0 on strict builds during validation of
5746	* segments.
5747	* @remarks No-long-jump zone!!!
5748	*/
5749	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5750	{
5751	int rc = VERR_INTERNAL_ERROR_5;
5752	PVM pVM = pVCpu->CTX_SUFF(pVM);
5753	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5754	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5755
5756	/*
5757	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5758	*/
5759	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5760	{
5761	#ifdef VBOX_WITH_REM
5762	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5763	{
5764	Assert(!pVmxTransient->fIsNestedGuest);
5765	Assert(pVM->hm.s.vmx.pRealModeTSS);
5766	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5767	if ( pVmcsInfo->fWasInRealMode
5768	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5769	{
5770	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5771	in real-mode (e.g. OpenBSD 4.0) */
5772	REMFlushTBs(pVM);
5773	Log4Func(("Switch to protected mode detected!\n"));
5774	pVmcsInfo->fWasInRealMode = false;
5775	}
5776	}
5777	#endif
5778	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5779	{
5780	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5781	if (pVmcsInfo->RealMode.fRealOnV86Active)
5782	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5783	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5784	AssertRCReturn(rc, rc);
5785	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5786	}
5787
5788	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5789	{
5790	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5791	if (pVmcsInfo->RealMode.fRealOnV86Active)
5792	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5793	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5794	AssertRCReturn(rc, rc);
5795	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5796	}
5797
5798	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5799	{
5800	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5801	if (pVmcsInfo->RealMode.fRealOnV86Active)
5802	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5803	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5804	AssertRCReturn(rc, rc);
5805	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5806	}
5807
5808	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5809	{
5810	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5811	if (pVmcsInfo->RealMode.fRealOnV86Active)
5812	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5813	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5814	AssertRCReturn(rc, rc);
5815	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5816	}
5817
5818	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5819	{
5820	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5821	if (pVmcsInfo->RealMode.fRealOnV86Active)
5822	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5823	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5824	AssertRCReturn(rc, rc);
5825	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5826	}
5827
5828	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5829	{
5830	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5831	if (pVmcsInfo->RealMode.fRealOnV86Active)
5832	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5833	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5834	AssertRCReturn(rc, rc);
5835	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5836	}
5837
5838	#ifdef VBOX_STRICT
5839	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5840	#endif
5841	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5842	pCtx->cs.Attr.u));
5843	}
5844
5845	/*
5846	* Guest TR.
5847	*/
5848	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5849	{
5850	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5851
5852	/*
5853	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5854	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5855	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5856	*/
5857	uint16_t u16Sel;
5858	uint32_t u32Limit;
5859	uint64_t u64Base;
5860	uint32_t u32AccessRights;
5861	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5862	{
5863	u16Sel = pCtx->tr.Sel;
5864	u32Limit = pCtx->tr.u32Limit;
5865	u64Base = pCtx->tr.u64Base;
5866	u32AccessRights = pCtx->tr.Attr.u;
5867	}
5868	else
5869	{
5870	Assert(!pVmxTransient->fIsNestedGuest);
5871	Assert(pVM->hm.s.vmx.pRealModeTSS);
5872	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5873
5874	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5875	RTGCPHYS GCPhys;
5876	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5877	AssertRCReturn(rc, rc);
5878
5879	X86DESCATTR DescAttr;
5880	DescAttr.u = 0;
5881	DescAttr.n.u1Present = 1;
5882	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5883
5884	u16Sel = 0;
5885	u32Limit = HM_VTX_TSS_SIZE;
5886	u64Base = GCPhys;
5887	u32AccessRights = DescAttr.u;
5888	}
5889
5890	/* Validate. */
5891	Assert(!(u16Sel & RT_BIT(2)));
5892	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5893	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5894	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5895	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5896	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5897	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5898	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5899	Assert( (u32Limit & 0xfff) == 0xfff
5900	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5901	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5902	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5903
5904	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5905	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5906	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5907	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5908	AssertRCReturn(rc, rc);
5909
5910	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5911	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5912	}
5913
5914	/*
5915	* Guest GDTR.
5916	*/
5917	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5918	{
5919	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5920
5921	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5922	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5923	AssertRCReturn(rc, rc);
5924
5925	/* Validate. */
5926	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5927
5928	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5929	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5930	}
5931
5932	/*
5933	* Guest LDTR.
5934	*/
5935	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5936	{
5937	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5938
5939	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5940	uint32_t u32Access;
5941	if ( !pVmxTransient->fIsNestedGuest
5942	&& !pCtx->ldtr.Attr.u)
5943	u32Access = X86DESCATTR_UNUSABLE;
5944	else
5945	u32Access = pCtx->ldtr.Attr.u;
5946
5947	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5948	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5949	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5950	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5951	AssertRCReturn(rc, rc);
5952
5953	/* Validate. */
5954	if (!(u32Access & X86DESCATTR_UNUSABLE))
5955	{
5956	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5957	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5958	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5959	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5960	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5961	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5962	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5963	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5964	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5965	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5966	}
5967
5968	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5969	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5970	}
5971
5972	/*
5973	* Guest IDTR.
5974	*/
5975	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5976	{
5977	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5978
5979	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5980	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5981	AssertRCReturn(rc, rc);
5982
5983	/* Validate. */
5984	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5985
5986	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5987	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5988	}
5989
5990	return VINF_SUCCESS;
5991	}
5992
5993
5994	/**
5995	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5996	* areas.
5997	*
5998	* These MSRs will automatically be loaded to the host CPU on every successful
5999	* VM-entry and stored from the host CPU on every successful VM-exit.
6000	*
6001	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6002	* actual host MSR values are not- updated here for performance reasons. See
6003	* hmR0VmxExportHostMsrs().
6004	*
6005	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6006	*
6007	* @returns VBox status code.
6008	* @param pVCpu The cross context virtual CPU structure.
6009	* @param pVmxTransient The VMX-transient structure.
6010	*
6011	* @remarks No-long-jump zone!!!
6012	*/
6013	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6014	{
6015	AssertPtr(pVCpu);
6016	AssertPtr(pVmxTransient);
6017
6018	PVM pVM = pVCpu->CTX_SUFF(pVM);
6019	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6020
6021	/*
6022	* MSRs that we use the auto-load/store MSR area in the VMCS.
6023	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6024	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6025	*
6026	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6027	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6028	* emulation, nothing to do here.
6029	*/
6030	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6031	{
6032	if ( !pVmxTransient->fIsNestedGuest
6033	&& pVM->hm.s.fAllow64BitGuests)
6034	{
6035	#if HC_ARCH_BITS == 32
6036	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6037	Assert(!pVmxTransient->fIsNestedGuest);
6038
6039	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6040	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6041	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6042	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6043	AssertRCReturn(rc, rc);
6044	#endif
6045	}
6046	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6047	}
6048
6049	/*
6050	* Guest Sysenter MSRs.
6051	*/
6052	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6053	{
6054	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6055
6056	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6057	{
6058	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6059	AssertRCReturn(rc, rc);
6060	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6061	}
6062
6063	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6064	{
6065	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6066	AssertRCReturn(rc, rc);
6067	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6068	}
6069
6070	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6071	{
6072	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6073	AssertRCReturn(rc, rc);
6074	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6075	}
6076	}
6077
6078	/*
6079	* Guest/host EFER MSR.
6080	*/
6081	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6082	{
6083	/* Whether we are using the VMCS to swap the EFER MSR must have been
6084	determined earlier while exporting VM-entry/VM-exit controls. */
6085	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6086	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6087
6088	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6089	{
6090	/*
6091	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6092	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6093	*/
6094	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6095	{
6096	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6097	AssertRCReturn(rc, rc);
6098	}
6099	else
6100	{
6101	/*
6102	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6103	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6104	*/
6105	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6106	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6107	AssertRCReturn(rc, rc);
6108	}
6109	}
6110	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6111	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6112
6113	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6114	}
6115
6116	/*
6117	* Other MSRs.
6118	* Speculation Control (R/W).
6119	*/
6120	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6121	{
6122	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6123	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6124	{
6125	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6126	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6127	AssertRCReturn(rc, rc);
6128	}
6129	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6130	}
6131
6132	return VINF_SUCCESS;
6133	}
6134
6135
6136	/**
6137	* Selects up the appropriate function to run guest code.
6138	*
6139	* @returns VBox status code.
6140	* @param pVCpu The cross context virtual CPU structure.
6141	* @param pVmxTransient The VMX-transient structure.
6142	*
6143	* @remarks No-long-jump zone!!!
6144	*/
6145	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6146	{
6147	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6148	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6149
6150	if (CPUMIsGuestInLongModeEx(pCtx))
6151	{
6152	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6153	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6154	#endif
6155	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6156	#if HC_ARCH_BITS == 32
6157	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6158	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6159	{
6160	#ifdef VBOX_STRICT
6161	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6162	{
6163	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6164	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6165	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6166	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6167	("fCtxChanged=%#RX64\n", fCtxChanged));
6168	}
6169	#endif
6170	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6171
6172	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6173	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6174	pVmcsInfo->fSwitchedTo64on32 = true;
6175	Log4Func(("Selected 64-bit switcher\n"));
6176	}
6177	#else
6178	/* 64-bit host. */
6179	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6180	#endif
6181	}
6182	else
6183	{
6184	/* Guest is not in long mode, use the 32-bit handler. */
6185	#if HC_ARCH_BITS == 32
6186	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6187	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6188	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6189	{
6190	# ifdef VBOX_STRICT
6191	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6192	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6193	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6194	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6195	("fCtxChanged=%#RX64\n", fCtxChanged));
6196	# endif
6197	}
6198	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6199	/*
6200	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6201	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6202	* switcher flag now because we know the guest is in a sane state where it's safe
6203	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6204	* the much faster 32-bit switcher again.
6205	*/
6206	if (!pVmcsInfo->fSwitchedTo64on32)
6207	{
6208	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6209	Log4Func(("Selected 32-bit switcher\n"));
6210	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6211	}
6212	else
6213	{
6214	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6215	if ( pVmcsInfo->RealMode.fRealOnV86Active
6216	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6217	{
6218	pVmcsInfo->fSwitchedTo64on32 = false;
6219	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6220	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6221	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6222	\| HM_CHANGED_HOST_CONTEXT);
6223	Log4Func(("Selected 32-bit switcher (safe)\n"));
6224	}
6225	}
6226	# else
6227	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6228	# endif
6229	#else
6230	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6231	#endif
6232	}
6233	Assert(pVmcsInfo->pfnStartVM);
6234	return VINF_SUCCESS;
6235	}
6236
6237
6238	/**
6239	* Wrapper for running the guest code in VT-x.
6240	*
6241	* @returns VBox status code, no informational status codes.
6242	* @param pVCpu The cross context virtual CPU structure.
6243	* @param pVmxTransient The VMX-transient structure.
6244	*
6245	* @remarks No-long-jump zone!!!
6246	*/
6247	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6248	{
6249	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6250	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6251	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6252
6253	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6254
6255	/*
6256	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6257	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6258	* callee-saved and thus the need for this XMM wrapper.
6259	*
6260	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6261	*/
6262	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6263	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6264	PVM pVM = pVCpu->CTX_SUFF(pVM);
6265	#ifdef VBOX_WITH_KERNEL_USING_XMM
6266	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6267	#else
6268	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6269	#endif
6270	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6271	return rc;
6272	}
6273
6274
6275	/**
6276	* Reports world-switch error and dumps some useful debug info.
6277	*
6278	* @param pVCpu The cross context virtual CPU structure.
6279	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6280	* @param pVmxTransient The VMX-transient structure (only
6281	* exitReason updated).
6282	*/
6283	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6284	{
6285	Assert(pVCpu);
6286	Assert(pVmxTransient);
6287	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6288
6289	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6290	switch (rcVMRun)
6291	{
6292	case VERR_VMX_INVALID_VMXON_PTR:
6293	AssertFailed();
6294	break;
6295	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6296	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6297	{
6298	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6299	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6300	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6301	AssertRC(rc);
6302
6303	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6304	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6305	Cannot do it here as we may have been long preempted. */
6306
6307	#ifdef VBOX_STRICT
6308	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6309	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6310	pVmxTransient->uExitReason));
6311	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6312	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6313	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6314	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6315	else
6316	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6317	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6318	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6319
6320	/* VMX control bits. */
6321	uint32_t u32Val;
6322	uint64_t u64Val;
6323	RTHCUINTREG uHCReg;
6324	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6325	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6326	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6327	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6328	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6329	{
6330	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6331	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6332	}
6333	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6334	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6335	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6336	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6337	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6338	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6339	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6340	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6341	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6342	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6343	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6344	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6345	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6346	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6347	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6348	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6349	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6350	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6351	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6352	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6353	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6354	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6355	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6356	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6357	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6358	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6359	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6360	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6361	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6362	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6363	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6364	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6365	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6366	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6367	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6368	{
6369	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6370	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6371	}
6372
6373	/* Guest bits. */
6374	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6375	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6376	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6377	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6378	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6379	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6380	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6381	{
6382	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6383	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6384	}
6385
6386	/* Host bits. */
6387	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6388	Log4(("Host CR0 %#RHr\n", uHCReg));
6389	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6390	Log4(("Host CR3 %#RHr\n", uHCReg));
6391	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6392	Log4(("Host CR4 %#RHr\n", uHCReg));
6393
6394	RTGDTR HostGdtr;
6395	PCX86DESCHC pDesc;
6396	ASMGetGDTR(&HostGdtr);
6397	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6398	Log4(("Host CS %#08x\n", u32Val));
6399	if (u32Val < HostGdtr.cbGdt)
6400	{
6401	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6402	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6403	}
6404
6405	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6406	Log4(("Host DS %#08x\n", u32Val));
6407	if (u32Val < HostGdtr.cbGdt)
6408	{
6409	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6410	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6411	}
6412
6413	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6414	Log4(("Host ES %#08x\n", u32Val));
6415	if (u32Val < HostGdtr.cbGdt)
6416	{
6417	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6418	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6419	}
6420
6421	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6422	Log4(("Host FS %#08x\n", u32Val));
6423	if (u32Val < HostGdtr.cbGdt)
6424	{
6425	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6426	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6427	}
6428
6429	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6430	Log4(("Host GS %#08x\n", u32Val));
6431	if (u32Val < HostGdtr.cbGdt)
6432	{
6433	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6434	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6435	}
6436
6437	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6438	Log4(("Host SS %#08x\n", u32Val));
6439	if (u32Val < HostGdtr.cbGdt)
6440	{
6441	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6442	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6443	}
6444
6445	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6446	Log4(("Host TR %#08x\n", u32Val));
6447	if (u32Val < HostGdtr.cbGdt)
6448	{
6449	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6450	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6451	}
6452
6453	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6454	Log4(("Host TR Base %#RHv\n", uHCReg));
6455	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6456	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6457	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6458	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6459	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6460	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6461	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6462	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6463	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6464	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6465	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6466	Log4(("Host RSP %#RHv\n", uHCReg));
6467	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6468	Log4(("Host RIP %#RHv\n", uHCReg));
6469	# if HC_ARCH_BITS == 64
6470	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6471	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6472	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6473	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6474	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6475	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6476	# endif
6477	#endif /* VBOX_STRICT */
6478	break;
6479	}
6480
6481	default:
6482	/* Impossible */
6483	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6484	break;
6485	}
6486	}
6487
6488
6489	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6490	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6491	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6492	# endif
6493
6494	/**
6495	* Initialize the VMCS-Read cache.
6496	*
6497	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6498	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6499	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6500	* (those that have a 32-bit FULL & HIGH part).
6501	*
6502	* @param pVCpu The cross context virtual CPU structure.
6503	*/
6504	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6505	{
6506	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6507	do { \
6508	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6509	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6510	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6511	++cReadFields; \
6512	} while (0)
6513
6514	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6515	uint32_t cReadFields = 0;
6516
6517	/*
6518	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6519	* and serve to indicate exceptions to the rules.
6520	*/
6521
6522	/* Guest-natural selector base fields. */
6523	#if 0
6524	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6527	#endif
6528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6530	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6531	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6532	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6533	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6534	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6535	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6536	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6537	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6538	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6539	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6540	#if 0
6541	/* Unused natural width guest-state fields. */
6542	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6543	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6544	#endif
6545	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6546	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6547
6548	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6549	these 64-bit fields (using "FULL" and "HIGH" fields). */
6550	#if 0
6551	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6552	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6553	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6554	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6555	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6556	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6557	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6558	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6559	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6560	#endif
6561
6562	/* Natural width guest-state fields. */
6563	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6564	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6565
6566	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6567	{
6568	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6569	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6570	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6571	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6572	}
6573	else
6574	{
6575	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6576	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6577	}
6578
6579	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6580	}
6581
6582
6583	/**
6584	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6585	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6586	* darwin, running 64-bit guests).
6587	*
6588	* @returns VBox status code.
6589	* @param pVCpu The cross context virtual CPU structure.
6590	* @param idxField The VMCS field encoding.
6591	* @param u64Val 16, 32 or 64-bit value.
6592	*/
6593	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6594	{
6595	AssertPtr(pVCpu);
6596	int rc;
6597	switch (idxField)
6598	{
6599	/*
6600	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6601	*/
6602	/* 64-bit Control fields. */
6603	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6604	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6605	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6606	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6607	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6608	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6609	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6610	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6611	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6612	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6613	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6614	case VMX_VMCS64_CTRL_EPTP_FULL:
6615	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6616	/* 64-bit Guest-state fields. */
6617	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6618	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6619	case VMX_VMCS64_GUEST_PAT_FULL:
6620	case VMX_VMCS64_GUEST_EFER_FULL:
6621	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6622	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6623	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6624	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6625	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6626	/* 64-bit Host-state fields. */
6627	case VMX_VMCS64_HOST_PAT_FULL:
6628	case VMX_VMCS64_HOST_EFER_FULL:
6629	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6630	{
6631	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6632	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6633	break;
6634	}
6635
6636	/*
6637	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6638	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6639	*/
6640	/* Natural-width Guest-state fields. */
6641	case VMX_VMCS_GUEST_CR3:
6642	case VMX_VMCS_GUEST_ES_BASE:
6643	case VMX_VMCS_GUEST_CS_BASE:
6644	case VMX_VMCS_GUEST_SS_BASE:
6645	case VMX_VMCS_GUEST_DS_BASE:
6646	case VMX_VMCS_GUEST_FS_BASE:
6647	case VMX_VMCS_GUEST_GS_BASE:
6648	case VMX_VMCS_GUEST_LDTR_BASE:
6649	case VMX_VMCS_GUEST_TR_BASE:
6650	case VMX_VMCS_GUEST_GDTR_BASE:
6651	case VMX_VMCS_GUEST_IDTR_BASE:
6652	case VMX_VMCS_GUEST_RSP:
6653	case VMX_VMCS_GUEST_RIP:
6654	case VMX_VMCS_GUEST_SYSENTER_ESP:
6655	case VMX_VMCS_GUEST_SYSENTER_EIP:
6656	{
6657	if (!(RT_HI_U32(u64Val)))
6658	{
6659	/* If this field is 64-bit, VT-x will zero out the top bits. */
6660	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6661	}
6662	else
6663	{
6664	/* Assert that only the 32->64 switcher case should ever come here. */
6665	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6666	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6667	}
6668	break;
6669	}
6670
6671	default:
6672	{
6673	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6674	pVCpu->hm.s.u32HMError = idxField;
6675	rc = VERR_INVALID_PARAMETER;
6676	break;
6677	}
6678	}
6679	AssertRCReturn(rc, rc);
6680	return rc;
6681	}
6682
6683
6684	/**
6685	* Queue up a VMWRITE by using the VMCS write cache.
6686	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6687	*
6688	* @param pVCpu The cross context virtual CPU structure.
6689	* @param idxField The VMCS field encoding.
6690	* @param u64Val 16, 32 or 64-bit value.
6691	*/
6692	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6693	{
6694	AssertPtr(pVCpu);
6695	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6696
6697	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6698	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6699
6700	/* Make sure there are no duplicates. */
6701	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6702	{
6703	if (pCache->Write.aField[i] == idxField)
6704	{
6705	pCache->Write.aFieldVal[i] = u64Val;
6706	return VINF_SUCCESS;
6707	}
6708	}
6709
6710	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6711	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6712	pCache->Write.cValidEntries++;
6713	return VINF_SUCCESS;
6714	}
6715	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6716
6717
6718	/**
6719	* Sets up the usage of TSC-offsetting and updates the VMCS.
6720	*
6721	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6722	* VMX-preemption timer.
6723	*
6724	* @returns VBox status code.
6725	* @param pVCpu The cross context virtual CPU structure.
6726	* @param pVmxTransient The VMX-transient structure.
6727	*
6728	* @remarks No-long-jump zone!!!
6729	*/
6730	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6731	{
6732	bool fOffsettedTsc;
6733	bool fParavirtTsc;
6734	uint64_t uTscOffset;
6735	PVM pVM = pVCpu->CTX_SUFF(pVM);
6736	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6737
6738	if (pVM->hm.s.vmx.fUsePreemptTimer)
6739	{
6740	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6741
6742	/* Make sure the returned values have sane upper and lower boundaries. */
6743	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6744	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6745	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6746	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6747
6748	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6749	* preemption timers here. We probably need to clamp the preemption timer,
6750	* after converting the timer value to the host. */
6751	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6752	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6753	AssertRC(rc);
6754	}
6755	else
6756	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6757
6758	if (fParavirtTsc)
6759	{
6760	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6761	information before every VM-entry, hence disable it for performance sake. */
6762	#if 0
6763	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6764	AssertRC(rc);
6765	#endif
6766	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6767	}
6768
6769	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6770	if ( fOffsettedTsc
6771	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6772	{
6773	if (pVmxTransient->fIsNestedGuest)
6774	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6775	if (pVmcsInfo->u64TscOffset != uTscOffset)
6776	{
6777	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6778	AssertRC(rc);
6779	pVmcsInfo->u64TscOffset = uTscOffset;
6780	}
6781
6782	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6783	{
6784	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6785	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6786	AssertRC(rc);
6787	pVmcsInfo->u32ProcCtls = uProcCtls;
6788	}
6789	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6790	}
6791	else
6792	{
6793	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6794	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6795	{
6796	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6797	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6798	AssertRC(rc);
6799	pVmcsInfo->u32ProcCtls = uProcCtls;
6800	}
6801	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6802	}
6803	}
6804
6805
6806	/**
6807	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6808	* VM-exit interruption info type.
6809	*
6810	* @returns The IEM exception flags.
6811	* @param uVector The event vector.
6812	* @param uVmxEventType The VMX event type.
6813	*
6814	* @remarks This function currently only constructs flags required for
6815	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6816	* and CR2 aspects of an exception are not included).
6817	*/
6818	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6819	{
6820	uint32_t fIemXcptFlags;
6821	switch (uVmxEventType)
6822	{
6823	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6824	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6825	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6826	break;
6827
6828	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6829	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6830	break;
6831
6832	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6833	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6834	break;
6835
6836	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6837	{
6838	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6839	if (uVector == X86_XCPT_BP)
6840	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6841	else if (uVector == X86_XCPT_OF)
6842	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6843	else
6844	{
6845	fIemXcptFlags = 0;
6846	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6847	}
6848	break;
6849	}
6850
6851	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6852	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6853	break;
6854
6855	default:
6856	fIemXcptFlags = 0;
6857	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6858	break;
6859	}
6860	return fIemXcptFlags;
6861	}
6862
6863
6864	/**
6865	* Sets an event as a pending event to be injected into the guest.
6866	*
6867	* @param pVCpu The cross context virtual CPU structure.
6868	* @param u32IntInfo The VM-entry interruption-information field.
6869	* @param cbInstr The VM-entry instruction length in bytes (for software
6870	* interrupts, exceptions and privileged software
6871	* exceptions).
6872	* @param u32ErrCode The VM-entry exception error code.
6873	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6874	* page-fault.
6875	*/
6876	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6877	RTGCUINTPTR GCPtrFaultAddress)
6878	{
6879	Assert(!pVCpu->hm.s.Event.fPending);
6880	pVCpu->hm.s.Event.fPending = true;
6881	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6882	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6883	pVCpu->hm.s.Event.cbInstr = cbInstr;
6884	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6885	}
6886
6887
6888	/**
6889	* Sets an external interrupt as pending-for-injection into the VM.
6890	*
6891	* @param pVCpu The cross context virtual CPU structure.
6892	* @param u8Interrupt The external interrupt vector.
6893	*/
6894	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6895	{
6896	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6897	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6898	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6899	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6900	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6901	}
6902
6903
6904	/**
6905	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6906	*
6907	* @param pVCpu The cross context virtual CPU structure.
6908	*/
6909	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6910	{
6911	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6912	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6913	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6914	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6915	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6916	}
6917
6918
6919	/**
6920	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6921	*
6922	* @param pVCpu The cross context virtual CPU structure.
6923	*/
6924	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6925	{
6926	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6927	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6928	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6929	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6930	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6931	}
6932
6933
6934	/**
6935	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6936	*
6937	* @param pVCpu The cross context virtual CPU structure.
6938	*/
6939	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6940	{
6941	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6942	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6943	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6944	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6945	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6946	}
6947
6948
6949	/**
6950	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6951	*
6952	* @param pVCpu The cross context virtual CPU structure.
6953	*/
6954	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6955	{
6956	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6957	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6958	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6959	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6960	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6961	}
6962
6963
6964	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6965	/**
6966	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6967	*
6968	* @param pVCpu The cross context virtual CPU structure.
6969	* @param u32ErrCode The error code for the general-protection exception.
6970	*/
6971	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6972	{
6973	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6974	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6975	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6976	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6977	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6978	}
6979
6980
6981	/**
6982	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6983	*
6984	* @param pVCpu The cross context virtual CPU structure.
6985	* @param u32ErrCode The error code for the stack exception.
6986	*/
6987	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6988	{
6989	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6990	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6991	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6992	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6993	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6994	}
6995
6996
6997	/**
6998	* Decodes the memory operand of an instruction that caused a VM-exit.
6999	*
7000	* The Exit qualification field provides the displacement field for memory
7001	* operand instructions, if any.
7002	*
7003	* @returns Strict VBox status code (i.e. informational status codes too).
7004	* @retval VINF_SUCCESS if the operand was successfully decoded.
7005	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
7006	* operand.
7007	* @param pVCpu The cross context virtual CPU structure.
7008	* @param uExitInstrInfo The VM-exit instruction information field.
7009	* @param enmMemAccess The memory operand's access type (read or write).
7010	* @param GCPtrDisp The instruction displacement field, if any. For
7011	* RIP-relative addressing pass RIP + displacement here.
7012	* @param pGCPtrMem Where to store the effective destination memory address.
7013	*
7014	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
7015	* virtual-8086 mode hence skips those checks while verifying if the
7016	* segment is valid.
7017	*/
7018	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
7019	PRTGCPTR pGCPtrMem)
7020	{
7021	Assert(pGCPtrMem);
7022	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
7023	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
7024	\| CPUMCTX_EXTRN_CR0);
7025
7026	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
7027	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
7028	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
7029
7030	VMXEXITINSTRINFO ExitInstrInfo;
7031	ExitInstrInfo.u = uExitInstrInfo;
7032	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
7033	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
7034	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
7035	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
7036	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
7037	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
7038	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
7039	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
7040	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
7041
7042	/*
7043	* Validate instruction information.
7044	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
7045	*/
7046	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
7047	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
7048	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
7049	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
7050	AssertLogRelMsgReturn(fIsMemOperand,
7051	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
7052
7053	/*
7054	* Compute the complete effective address.
7055	*
7056	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
7057	* See AMD spec. 4.5.2 "Segment Registers".
7058	*/
7059	RTGCPTR GCPtrMem = GCPtrDisp;
7060	if (fBaseRegValid)
7061	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
7062	if (fIdxRegValid)
7063	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
7064
7065	RTGCPTR const GCPtrOff = GCPtrMem;
7066	if ( !fIsLongMode
7067	\|\| iSegReg >= X86_SREG_FS)
7068	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
7069	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
7070
7071	/*
7072	* Validate effective address.
7073	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
7074	*/
7075	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
7076	Assert(cbAccess > 0);
7077	if (fIsLongMode)
7078	{
7079	if (X86_IS_CANONICAL(GCPtrMem))
7080	{
7081	*pGCPtrMem = GCPtrMem;
7082	return VINF_SUCCESS;
7083	}
7084
7085	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
7086	* "Data Limit Checks in 64-bit Mode". */
7087	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
7088	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7089	return VINF_HM_PENDING_XCPT;
7090	}
7091
7092	/*
7093	* This is a watered down version of iemMemApplySegment().
7094	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
7095	* and segment CPL/DPL checks are skipped.
7096	*/
7097	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
7098	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
7099	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7100
7101	/* Check if the segment is present and usable. */
7102	if ( pSel->Attr.n.u1Present
7103	&& !pSel->Attr.n.u1Unusable)
7104	{
7105	Assert(pSel->Attr.n.u1DescType);
7106	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
7107	{
7108	/* Check permissions for the data segment. */
7109	if ( enmMemAccess == VMXMEMACCESS_WRITE
7110	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
7111	{
7112	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7113	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
7114	return VINF_HM_PENDING_XCPT;
7115	}
7116
7117	/* Check limits if it's a normal data segment. */
7118	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
7119	{
7120	if ( GCPtrFirst32 > pSel->u32Limit
7121	\|\| GCPtrLast32 > pSel->u32Limit)
7122	{
7123	Log4Func(("Data segment limit exceeded. "
7124	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
7125	GCPtrLast32, pSel->u32Limit));
7126	if (iSegReg == X86_SREG_SS)
7127	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7128	else
7129	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7130	return VINF_HM_PENDING_XCPT;
7131	}
7132	}
7133	else
7134	{
7135	/* Check limits if it's an expand-down data segment.
7136	Note! The upper boundary is defined by the B bit, not the G bit! */
7137	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
7138	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
7139	{
7140	Log4Func(("Expand-down data segment limit exceeded. "
7141	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
7142	GCPtrLast32, pSel->u32Limit));
7143	if (iSegReg == X86_SREG_SS)
7144	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7145	else
7146	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7147	return VINF_HM_PENDING_XCPT;
7148	}
7149	}
7150	}
7151	else
7152	{
7153	/* Check permissions for the code segment. */
7154	if ( enmMemAccess == VMXMEMACCESS_WRITE
7155	\|\| ( enmMemAccess == VMXMEMACCESS_READ
7156	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
7157	{
7158	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
7159	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7160	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7161	return VINF_HM_PENDING_XCPT;
7162	}
7163
7164	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
7165	if ( GCPtrFirst32 > pSel->u32Limit
7166	\|\| GCPtrLast32 > pSel->u32Limit)
7167	{
7168	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
7169	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
7170	if (iSegReg == X86_SREG_SS)
7171	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7172	else
7173	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7174	return VINF_HM_PENDING_XCPT;
7175	}
7176	}
7177	}
7178	else
7179	{
7180	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7181	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7182	return VINF_HM_PENDING_XCPT;
7183	}
7184
7185	*pGCPtrMem = GCPtrMem;
7186	return VINF_SUCCESS;
7187	}
7188
7189
7190	/**
7191	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
7192	* guest attempting to execute a VMX instruction.
7193	*
7194	* @returns Strict VBox status code (i.e. informational status codes too).
7195	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
7196	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
7197	*
7198	* @param pVCpu The cross context virtual CPU structure.
7199	* @param uExitReason The VM-exit reason.
7200	*
7201	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
7202	* @remarks No-long-jump zone!!!
7203	*/
7204	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
7205	{
7206	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
7207	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7208
7209	/*
7210	* The physical CPU would have already checked the CPU mode/code segment.
7211	* We shall just assert here for paranoia.
7212	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7213	*/
7214	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7215	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
7216	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
7217
7218	if (uExitReason == VMX_EXIT_VMXON)
7219	{
7220	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7221
7222	/*
7223	* We check CR4.VMXE because it is required to be always set while in VMX operation
7224	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
7225	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7226	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7227	*/
7228	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7229	{
7230	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7231	hmR0VmxSetPendingXcptUD(pVCpu);
7232	return VINF_HM_PENDING_XCPT;
7233	}
7234	}
7235	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7236	{
7237	/*
7238	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7239	* (other than VMXON), we need to raise a #UD.
7240	*/
7241	Log4Func(("Not in VMX root mode -> #UD\n"));
7242	hmR0VmxSetPendingXcptUD(pVCpu);
7243	return VINF_HM_PENDING_XCPT;
7244	}
7245
7246	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7247	return VINF_SUCCESS;
7248	}
7249	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7250
7251
7252	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7253	{
7254	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7255
7256	/*
7257	* If VT-x marks the segment as unusable, most other bits remain undefined:
7258	* - For CS the L, D and G bits have meaning.
7259	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7260	* - For the remaining data segments no bits are defined.
7261	*
7262	* The present bit and the unusable bit has been observed to be set at the
7263	* same time (the selector was supposed to be invalid as we started executing
7264	* a V8086 interrupt in ring-0).
7265	*
7266	* What should be important for the rest of the VBox code, is that the P bit is
7267	* cleared. Some of the other VBox code recognizes the unusable bit, but
7268	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7269	* safe side here, we'll strip off P and other bits we don't care about. If
7270	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7271	*
7272	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7273	*/
7274	#ifdef VBOX_STRICT
7275	uint32_t const uAttr = pSelReg->Attr.u;
7276	#endif
7277
7278	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7279	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7280	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7281
7282	#ifdef VBOX_STRICT
7283	VMMRZCallRing3Disable(pVCpu);
7284	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7285	# ifdef DEBUG_bird
7286	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7287	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7288	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7289	# endif
7290	VMMRZCallRing3Enable(pVCpu);
7291	NOREF(uAttr);
7292	#endif
7293	RT_NOREF2(pVCpu, idxSel);
7294	}
7295
7296
7297	/**
7298	* Imports a guest segment register from the current VMCS into the guest-CPU
7299	* context.
7300	*
7301	* @returns VBox status code.
7302	* @param pVCpu The cross context virtual CPU structure.
7303	* @param iSegReg The segment register number (X86_SREG_XXX).
7304	*
7305	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7306	* do not log!
7307	*/
7308	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7309	{
7310	Assert(iSegReg < X86_SREG_COUNT);
7311
7312	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7313	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7314	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7315	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7316	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7317	#else
7318	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7319	#endif
7320	uint64_t u64Base;
7321	uint32_t u32Sel, u32Limit, u32Attr;
7322	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7323	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7324	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7325	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7326	if (RT_SUCCESS(rc))
7327	{
7328	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7329	pSelReg->Sel = u32Sel;
7330	pSelReg->ValidSel = u32Sel;
7331	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7332	pSelReg->u32Limit = u32Limit;
7333	pSelReg->u64Base = u64Base;
7334	pSelReg->Attr.u = u32Attr;
7335	if (u32Attr & X86DESCATTR_UNUSABLE)
7336	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7337	}
7338	return rc;
7339	}
7340
7341
7342	/**
7343	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7344	*
7345	* @returns VBox status code.
7346	* @param pVCpu The cross context virtual CPU structure.
7347	*
7348	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7349	* do not log!
7350	*/
7351	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7352	{
7353	uint64_t u64Base;
7354	uint32_t u32Sel, u32Limit, u32Attr;
7355	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7356	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7357	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7358	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7359	if (RT_SUCCESS(rc))
7360	{
7361	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7362	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7363	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7364	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7365	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7366	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7367	if (u32Attr & X86DESCATTR_UNUSABLE)
7368	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7369	}
7370	return rc;
7371	}
7372
7373
7374	/**
7375	* Imports the guest TR from the current VMCS into the guest-CPU context.
7376	*
7377	* @returns VBox status code.
7378	* @param pVCpu The cross context virtual CPU structure.
7379	*
7380	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7381	* do not log!
7382	*/
7383	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7384	{
7385	uint32_t u32Sel, u32Limit, u32Attr;
7386	uint64_t u64Base;
7387	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7388	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7389	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7390	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7391	AssertRCReturn(rc, rc);
7392
7393	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7394	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7395	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7396	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7397	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7398	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7399	/* TR is the only selector that can never be unusable. */
7400	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7401	return VINF_SUCCESS;
7402	}
7403
7404
7405	/**
7406	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7407	*
7408	* @returns VBox status code.
7409	* @param pVCpu The cross context virtual CPU structure.
7410	*
7411	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7412	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7413	* instead!!!
7414	*/
7415	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7416	{
7417	uint64_t u64Val;
7418	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7419	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7420	{
7421	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7422	if (RT_SUCCESS(rc))
7423	{
7424	pCtx->rip = u64Val;
7425	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7426	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7427	}
7428	return rc;
7429	}
7430	return VINF_SUCCESS;
7431	}
7432
7433
7434	/**
7435	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7436	*
7437	* @returns VBox status code.
7438	* @param pVCpu The cross context virtual CPU structure.
7439	* @param pVmcsInfo The VMCS info. object.
7440	*
7441	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7442	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7443	* instead!!!
7444	*/
7445	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7446	{
7447	uint32_t u32Val;
7448	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7449	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7450	{
7451	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7452	if (RT_SUCCESS(rc))
7453	{
7454	pCtx->eflags.u32 = u32Val;
7455
7456	/* Restore eflags for real-on-v86-mode hack. */
7457	if (pVmcsInfo->RealMode.fRealOnV86Active)
7458	{
7459	pCtx->eflags.Bits.u1VM = 0;
7460	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7461	}
7462	}
7463	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7464	return rc;
7465	}
7466	return VINF_SUCCESS;
7467	}
7468
7469
7470	/**
7471	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7472	* context.
7473	*
7474	* @returns VBox status code.
7475	* @param pVCpu The cross context virtual CPU structure.
7476	* @param pVmcsInfo The VMCS info. object.
7477	*
7478	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7479	* do not log!
7480	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7481	* instead!!!
7482	*/
7483	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7484	{
7485	uint32_t u32Val;
7486	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7487	if (RT_SUCCESS(rc))
7488	{
7489	if (!u32Val)
7490	{
7491	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7492	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7493
7494	CPUMSetGuestNmiBlocking(pVCpu, false);
7495	}
7496	else
7497	{
7498	/*
7499	* We must import RIP here to set our EM interrupt-inhibited state.
7500	* We also import RFLAGS as our code that evaluates pending interrupts
7501	* before VM-entry requires it.
7502	*/
7503	rc = hmR0VmxImportGuestRip(pVCpu);
7504	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7505	if (RT_SUCCESS(rc))
7506	{
7507	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7508	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7509	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7510	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7511
7512	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7513	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7514	}
7515	}
7516	}
7517	return rc;
7518	}
7519
7520
7521	/**
7522	* Worker for VMXR0ImportStateOnDemand.
7523	*
7524	* @returns VBox status code.
7525	* @param pVCpu The cross context virtual CPU structure.
7526	* @param pVmcsInfo The VMCS info. object.
7527	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7528	*/
7529	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7530	{
7531	#define VMXLOCAL_BREAK_RC(a_rc) \
7532	if (RT_SUCCESS(a_rc)) \
7533	{ } \
7534	else \
7535	break
7536
7537	int rc = VINF_SUCCESS;
7538	PVM pVM = pVCpu->CTX_SUFF(pVM);
7539	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7540	uint64_t u64Val;
7541	uint32_t u32Val;
7542
7543	/*
7544	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7545	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7546	* neither are other host platforms.
7547	*
7548	* Committing this temporarily as it prevents BSOD.
7549	*
7550	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7551	*/
7552	#ifdef RT_OS_WINDOWS
7553	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7554	return VERR_HM_IPE_1;
7555	#endif
7556
7557	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7558
7559	/*
7560	* We disable interrupts to make the updating of the state and in particular
7561	* the fExtrn modification atomic wrt to preemption hooks.
7562	*/
7563	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7564
7565	fWhat &= pCtx->fExtrn;
7566	if (fWhat)
7567	{
7568	do
7569	{
7570	if (fWhat & CPUMCTX_EXTRN_RIP)
7571	{
7572	rc = hmR0VmxImportGuestRip(pVCpu);
7573	VMXLOCAL_BREAK_RC(rc);
7574	}
7575
7576	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7577	{
7578	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7579	VMXLOCAL_BREAK_RC(rc);
7580	}
7581
7582	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7583	{
7584	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7585	VMXLOCAL_BREAK_RC(rc);
7586	}
7587
7588	if (fWhat & CPUMCTX_EXTRN_RSP)
7589	{
7590	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7591	VMXLOCAL_BREAK_RC(rc);
7592	pCtx->rsp = u64Val;
7593	}
7594
7595	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7596	{
7597	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7598	if (fWhat & CPUMCTX_EXTRN_CS)
7599	{
7600	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7601	rc \|= hmR0VmxImportGuestRip(pVCpu);
7602	if (fRealOnV86Active)
7603	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7604	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7605	}
7606	if (fWhat & CPUMCTX_EXTRN_SS)
7607	{
7608	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7609	if (fRealOnV86Active)
7610	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7611	}
7612	if (fWhat & CPUMCTX_EXTRN_DS)
7613	{
7614	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7615	if (fRealOnV86Active)
7616	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7617	}
7618	if (fWhat & CPUMCTX_EXTRN_ES)
7619	{
7620	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7621	if (fRealOnV86Active)
7622	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7623	}
7624	if (fWhat & CPUMCTX_EXTRN_FS)
7625	{
7626	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7627	if (fRealOnV86Active)
7628	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7629	}
7630	if (fWhat & CPUMCTX_EXTRN_GS)
7631	{
7632	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7633	if (fRealOnV86Active)
7634	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7635	}
7636	VMXLOCAL_BREAK_RC(rc);
7637	}
7638
7639	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7640	{
7641	if (fWhat & CPUMCTX_EXTRN_LDTR)
7642	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7643
7644	if (fWhat & CPUMCTX_EXTRN_GDTR)
7645	{
7646	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7647	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7648	pCtx->gdtr.pGdt = u64Val;
7649	pCtx->gdtr.cbGdt = u32Val;
7650	}
7651
7652	/* Guest IDTR. */
7653	if (fWhat & CPUMCTX_EXTRN_IDTR)
7654	{
7655	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7656	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7657	pCtx->idtr.pIdt = u64Val;
7658	pCtx->idtr.cbIdt = u32Val;
7659	}
7660
7661	/* Guest TR. */
7662	if (fWhat & CPUMCTX_EXTRN_TR)
7663	{
7664	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7665	don't need to import that one. */
7666	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7667	rc \|= hmR0VmxImportGuestTr(pVCpu);
7668	}
7669	VMXLOCAL_BREAK_RC(rc);
7670	}
7671
7672	if (fWhat & CPUMCTX_EXTRN_DR7)
7673	{
7674	if (!pVCpu->hm.s.fUsingHyperDR7)
7675	{
7676	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7677	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7678	VMXLOCAL_BREAK_RC(rc);
7679	pCtx->dr[7] = u32Val;
7680	}
7681	}
7682
7683	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7684	{
7685	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7686	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7687	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7688	pCtx->SysEnter.cs = u32Val;
7689	VMXLOCAL_BREAK_RC(rc);
7690	}
7691
7692	#if HC_ARCH_BITS == 64
7693	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7694	{
7695	if ( pVM->hm.s.fAllow64BitGuests
7696	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7697	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7698	}
7699
7700	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7701	{
7702	if ( pVM->hm.s.fAllow64BitGuests
7703	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7704	{
7705	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7706	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7707	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7708	}
7709	}
7710	#endif
7711
7712	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7713	#if HC_ARCH_BITS == 32
7714	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7715	#endif
7716	)
7717	{
7718	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7719	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7720	Assert(pMsrs);
7721	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7722	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7723	for (uint32_t i = 0; i < cMsrs; i++)
7724	{
7725	uint32_t const idMsr = pMsrs[i].u32Msr;
7726	switch (idMsr)
7727	{
7728	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7729	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7730	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7731	#if HC_ARCH_BITS == 32
7732	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7733	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7734	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7735	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7736	#endif
7737	default:
7738	{
7739	pCtx->fExtrn = 0;
7740	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7741	ASMSetFlags(fEFlags);
7742	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7743	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7744	}
7745	}
7746	}
7747	}
7748
7749	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7750	{
7751	uint64_t u64Shadow;
7752	if (fWhat & CPUMCTX_EXTRN_CR0)
7753	{
7754	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7755	* remove when we drop 32-bit host w/ 64-bit host support, see
7756	* @bugref{9180#c39}. */
7757	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7758	#if HC_ARCH_BITS == 32
7759	uint32_t u32Shadow;
7760	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7761	u64Shadow = u32Shadow;
7762	#else
7763	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7764	#endif
7765	VMXLOCAL_BREAK_RC(rc);
7766	u64Val = u32Val;
7767	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7768	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7769	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7770	/*
7771	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
7772	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
7773	*/
7774	if (CPUMIsGuestInVmxNonRootMode(pCtx))
7775	{
7776	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
7777	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
7778	}
7779	#endif
7780	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7781	CPUMSetGuestCR0(pVCpu, u64Val);
7782	VMMRZCallRing3Enable(pVCpu);
7783	}
7784
7785	if (fWhat & CPUMCTX_EXTRN_CR4)
7786	{
7787	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7788	* remove when we drop 32-bit host w/ 64-bit host support, see
7789	* @bugref{9180#c39}. */
7790	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7791	#if HC_ARCH_BITS == 32
7792	uint32_t u32Shadow;
7793	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7794	u64Shadow = u32Shadow;
7795	#else
7796	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7797	#endif
7798	VMXLOCAL_BREAK_RC(rc);
7799	u64Val = u32Val;
7800	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7801	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7802	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7803	/*
7804	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
7805	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
7806	*/
7807	if (CPUMIsGuestInVmxNonRootMode(pCtx))
7808	{
7809	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
7810	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
7811	}
7812	#endif
7813	pCtx->cr4 = u64Val;
7814	}
7815
7816	if (fWhat & CPUMCTX_EXTRN_CR3)
7817	{
7818	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7819	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7820	\|\| ( pVM->hm.s.fNestedPaging
7821	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7822	{
7823	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7824	VMXLOCAL_BREAK_RC(rc);
7825	if (pCtx->cr3 != u64Val)
7826	{
7827	pCtx->cr3 = u64Val;
7828	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7829	}
7830
7831	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7832	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7833	if (CPUMIsGuestInPAEModeEx(pCtx))
7834	{
7835	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7836	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7837	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7838	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7839	VMXLOCAL_BREAK_RC(rc);
7840	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7841	}
7842	}
7843	}
7844
7845	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7846	# if 0
7847	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
7848	* VM-exit handlers as parameters. We would handle it differently when using
7849	* the fast path. */
7850	/*
7851	* The hardware virtualization state currently consists of VMCS fields that may be
7852	* modified by execution of the nested-guest (that are not part of the general
7853	* guest state) and is visible to guest software. Hence, it is technically part of
7854	* the guest-CPU state when executing a nested-guest.
7855	*/
7856	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7857	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7858	{
7859	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7860	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7861	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7862	VMXLOCAL_BREAK_RC(rc);
7863
7864	/*
7865	* VM-entry can fail due to invalid-guest state, machine-check events and
7866	* MSR loading failures. Other than VM-exit reason and Exit qualification
7867	* all other VMCS fields are left unmodified on VM-entry failure.
7868	*
7869	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7870	*/
7871	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7872	if (!fEntryFailed)
7873	{
7874	/*
7875	* Some notes on VMCS fields that may need importing when the fast path
7876	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7877	*
7878	* Requires fixing up when using hardware-assisted VMX:
7879	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7880	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7881	* - IDT-vectoring info: Think about this.
7882	* - IDT-vectoring error code: Think about this.
7883	*
7884	* Emulated:
7885	* - Guest-interruptiblity state: Derived from FFs and RIP.
7886	* - Guest pending debug exceptions: Derived from DR6.
7887	* - Guest activity state: Emulated from EM state.
7888	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7889	* - Entry-interrupt info: Emulated, cleared to 0.
7890	*/
7891	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7892	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7893	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7894	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7895	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7896	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7897	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7898	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7899	/** @todo NSTVMX: Save and adjust preemption timer value. */
7900	}
7901
7902	VMXLOCAL_BREAK_RC(rc);
7903	}
7904	# endif
7905	#endif
7906	}
7907	} while (0);
7908
7909	if (RT_SUCCESS(rc))
7910	{
7911	/* Update fExtrn. */
7912	pCtx->fExtrn &= ~fWhat;
7913
7914	/* If everything has been imported, clear the HM keeper bit. */
7915	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7916	{
7917	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7918	Assert(!pCtx->fExtrn);
7919	}
7920	}
7921	}
7922	else
7923	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7924
7925	ASMSetFlags(fEFlags);
7926
7927	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7928
7929	if (RT_SUCCESS(rc))
7930	{ /* likely */ }
7931	else
7932	return rc;
7933
7934	/*
7935	* Honor any pending CR3 updates.
7936	*
7937	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7938	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7939	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7940	*
7941	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7942	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7943	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7944	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7945	*
7946	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7947	*/
7948	if (VMMRZCallRing3IsEnabled(pVCpu))
7949	{
7950	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7951	{
7952	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7953	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7954	}
7955
7956	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7957	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7958
7959	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7960	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7961	}
7962
7963	return VINF_SUCCESS;
7964	#undef VMXLOCAL_BREAK_RC
7965	}
7966
7967
7968	/**
7969	* Saves the guest state from the VMCS into the guest-CPU context.
7970	*
7971	* @returns VBox status code.
7972	* @param pVCpu The cross context virtual CPU structure.
7973	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7974	*/
7975	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7976	{
7977	AssertPtr(pVCpu);
7978	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7979	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7980	}
7981
7982
7983	/**
7984	* Check per-VM and per-VCPU force flag actions that require us to go back to
7985	* ring-3 for one reason or another.
7986	*
7987	* @returns Strict VBox status code (i.e. informational status codes too)
7988	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7989	* ring-3.
7990	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7991	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7992	* interrupts)
7993	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7994	* all EMTs to be in ring-3.
7995	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7996	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7997	* to the EM loop.
7998	*
7999	* @param pVCpu The cross context virtual CPU structure.
8000	* @param fStepping Whether we are single-stepping the guest using the
8001	* hypervisor debugger.
8002	*/
8003	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
8004	{
8005	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8006
8007	/*
8008	* Update pending interrupts into the APIC's IRR.
8009	*/
8010	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8011	APICUpdatePendingInterrupts(pVCpu);
8012
8013	/*
8014	* Anything pending? Should be more likely than not if we're doing a good job.
8015	*/
8016	PVM pVM = pVCpu->CTX_SUFF(pVM);
8017	if ( !fStepping
8018	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
8019	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
8020	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
8021	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
8022	return VINF_SUCCESS;
8023
8024	/* Pending PGM C3 sync. */
8025	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
8026	{
8027	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8028	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
8029	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8030	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8031	if (rcStrict2 != VINF_SUCCESS)
8032	{
8033	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8034	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8035	return rcStrict2;
8036	}
8037	}
8038
8039	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8040	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8041	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8042	{
8043	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8044	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8045	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8046	return rc2;
8047	}
8048
8049	/* Pending VM request packets, such as hardware interrupts. */
8050	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8051	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8052	{
8053	Log4Func(("Pending VM request forcing us back to ring-3\n"));
8054	return VINF_EM_PENDING_REQUEST;
8055	}
8056
8057	/* Pending PGM pool flushes. */
8058	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8059	{
8060	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8061	return VINF_PGM_POOL_FLUSH_PENDING;
8062	}
8063
8064	/* Pending DMA requests. */
8065	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8066	{
8067	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8068	return VINF_EM_RAW_TO_R3;
8069	}
8070
8071	return VINF_SUCCESS;
8072	}
8073
8074
8075	/**
8076	* Converts any TRPM trap into a pending HM event. This is typically used when
8077	* entering from ring-3 (not longjmp returns).
8078	*
8079	* @param pVCpu The cross context virtual CPU structure.
8080	*/
8081	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8082	{
8083	Assert(TRPMHasTrap(pVCpu));
8084	Assert(!pVCpu->hm.s.Event.fPending);
8085
8086	uint8_t uVector;
8087	TRPMEVENT enmTrpmEvent;
8088	RTGCUINT uErrCode;
8089	RTGCUINTPTR GCPtrFaultAddress;
8090	uint8_t cbInstr;
8091
8092	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8093	AssertRC(rc);
8094
8095	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
8096	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
8097	if (enmTrpmEvent == TRPM_TRAP)
8098	{
8099	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
8100	* generated using INT1 (ICEBP). */
8101	switch (uVector)
8102	{
8103	case X86_XCPT_NMI:
8104	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8105	break;
8106
8107	case X86_XCPT_BP:
8108	case X86_XCPT_OF:
8109	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8110	break;
8111
8112	case X86_XCPT_PF:
8113	case X86_XCPT_DF:
8114	case X86_XCPT_TS:
8115	case X86_XCPT_NP:
8116	case X86_XCPT_SS:
8117	case X86_XCPT_GP:
8118	case X86_XCPT_AC:
8119	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
8120	RT_FALL_THRU();
8121	default:
8122	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8123	break;
8124	}
8125	}
8126	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
8127	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8128	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
8129	{
8130	switch (uVector)
8131	{
8132	case X86_XCPT_BP:
8133	case X86_XCPT_OF:
8134	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8135	break;
8136
8137	default:
8138	Assert(uVector == X86_XCPT_DB);
8139	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8140	break;
8141	}
8142	}
8143	else
8144	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
8145
8146	rc = TRPMResetTrap(pVCpu);
8147	AssertRC(rc);
8148	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8149	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8150
8151	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8152	}
8153
8154
8155	/**
8156	* Converts the pending HM event into a TRPM trap.
8157	*
8158	* @param pVCpu The cross context virtual CPU structure.
8159	*/
8160	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8161	{
8162	Assert(pVCpu->hm.s.Event.fPending);
8163
8164	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8165	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
8166	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
8167	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
8168
8169	/* If a trap was already pending, we did something wrong! */
8170	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8171
8172	/** @todo Use HMVmxEventToTrpmEventType() later. */
8173	TRPMEVENT enmTrapType;
8174	switch (uVectorType)
8175	{
8176	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
8177	enmTrapType = TRPM_HARDWARE_INT;
8178	break;
8179
8180	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
8181	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
8182	enmTrapType = TRPM_TRAP;
8183	break;
8184
8185	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
8186	Assert(uVector == X86_XCPT_DB);
8187	enmTrapType = TRPM_SOFTWARE_INT;
8188	break;
8189
8190	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
8191	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8192	enmTrapType = TRPM_SOFTWARE_INT;
8193	break;
8194
8195	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8196	enmTrapType = TRPM_SOFTWARE_INT;
8197	break;
8198
8199	default:
8200	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
8201	enmTrapType = TRPM_32BIT_HACK;
8202	break;
8203	}
8204
8205	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8206
8207	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8208	AssertRC(rc);
8209
8210	if (fErrorCodeValid)
8211	TRPMSetErrorCode(pVCpu, uErrorCode);
8212
8213	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8214	&& uVector == X86_XCPT_PF)
8215	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8216	else if (enmTrapType == TRPM_SOFTWARE_INT)
8217	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8218
8219	/* We're now done converting the pending event. */
8220	pVCpu->hm.s.Event.fPending = false;
8221	}
8222
8223
8224	/**
8225	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8226	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8227	*
8228	* @param pVCpu The cross context virtual CPU structure.
8229	* @param pVmcsInfo The VMCS info. object.
8230	*/
8231	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8232	{
8233	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8234	{
8235	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8236	{
8237	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8238	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8239	AssertRC(rc);
8240	}
8241	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8242	}
8243
8244
8245	/**
8246	* Clears the interrupt-window exiting control in the VMCS.
8247	*
8248	* @param pVmcsInfo The VMCS info. object.
8249	*/
8250	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8251	{
8252	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8253	{
8254	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8255	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8256	}
8257	return VINF_SUCCESS;
8258	}
8259
8260
8261	/**
8262	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8263	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8264	*
8265	* @param pVCpu The cross context virtual CPU structure.
8266	* @param pVmcsInfo The VMCS info. object.
8267	*/
8268	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8269	{
8270	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8271	{
8272	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8273	{
8274	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8275	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8276	AssertRC(rc);
8277	Log4Func(("Setup NMI-window exiting\n"));
8278	}
8279	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8280	}
8281
8282
8283	/**
8284	* Clears the NMI-window exiting control in the VMCS.
8285	*
8286	* @param pVmcsInfo The VMCS info. object.
8287	*/
8288	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8289	{
8290	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8291	{
8292	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8293	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8294	}
8295	return VINF_SUCCESS;
8296	}
8297
8298
8299	/**
8300	* Does the necessary state syncing before returning to ring-3 for any reason
8301	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8302	*
8303	* @returns VBox status code.
8304	* @param pVCpu The cross context virtual CPU structure.
8305	* @param fImportState Whether to import the guest state from the VMCS back
8306	* to the guest-CPU context.
8307	*
8308	* @remarks No-long-jmp zone!!!
8309	*/
8310	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8311	{
8312	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8313	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8314
8315	RTCPUID idCpu = RTMpCpuId();
8316	Log4Func(("HostCpuId=%u\n", idCpu));
8317
8318	/*
8319	* !!! IMPORTANT !!!
8320	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8321	*/
8322
8323	/* Save the guest state if necessary. */
8324	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8325	if (fImportState)
8326	{
8327	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8328	AssertRCReturn(rc, rc);
8329	}
8330
8331	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8332	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8333	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8334
8335	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8336	#ifdef VBOX_STRICT
8337	if (CPUMIsHyperDebugStateActive(pVCpu))
8338	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8339	#endif
8340	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8341	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8342	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8343
8344	#if HC_ARCH_BITS == 64
8345	/* Restore host-state bits that VT-x only restores partially. */
8346	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8347	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8348	{
8349	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8350	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8351	}
8352	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8353	#endif
8354
8355	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8356	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8357	{
8358	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8359	if (!fImportState)
8360	{
8361	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8362	AssertRCReturn(rc, rc);
8363	}
8364	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8365	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8366	}
8367	else
8368	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8369
8370	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8371	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8372
8373	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8374	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8375	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8376	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8377	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8378	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8379	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8380	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8381	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8382
8383	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8384
8385	/** @todo This partially defeats the purpose of having preemption hooks.
8386	* The problem is, deregistering the hooks should be moved to a place that
8387	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8388	* context.
8389	*/
8390	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8391	AssertRCReturn(rc, rc);
8392
8393	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8394	NOREF(idCpu);
8395	return VINF_SUCCESS;
8396	}
8397
8398
8399	/**
8400	* Leaves the VT-x session.
8401	*
8402	* @returns VBox status code.
8403	* @param pVCpu The cross context virtual CPU structure.
8404	*
8405	* @remarks No-long-jmp zone!!!
8406	*/
8407	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8408	{
8409	HM_DISABLE_PREEMPT(pVCpu);
8410	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8411	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8412	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8413
8414	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8415	and done this from the VMXR0ThreadCtxCallback(). */
8416	if (!pVCpu->hm.s.fLeaveDone)
8417	{
8418	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8419	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8420	pVCpu->hm.s.fLeaveDone = true;
8421	}
8422	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8423
8424	/*
8425	* !!! IMPORTANT !!!
8426	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8427	*/
8428
8429	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8430	/** @todo Deregistering here means we need to VMCLEAR always
8431	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8432	* for calling VMMR0ThreadCtxHookDisable here! */
8433	VMMR0ThreadCtxHookDisable(pVCpu);
8434
8435	/* Leave HM context. This takes care of local init (term). */
8436	int rc = HMR0LeaveCpu(pVCpu);
8437
8438	HM_RESTORE_PREEMPT();
8439	return rc;
8440	}
8441
8442
8443	/**
8444	* Does the necessary state syncing before doing a longjmp to ring-3.
8445	*
8446	* @returns VBox status code.
8447	* @param pVCpu The cross context virtual CPU structure.
8448	*
8449	* @remarks No-long-jmp zone!!!
8450	*/
8451	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8452	{
8453	return hmR0VmxLeaveSession(pVCpu);
8454	}
8455
8456
8457	/**
8458	* Take necessary actions before going back to ring-3.
8459	*
8460	* An action requires us to go back to ring-3. This function does the necessary
8461	* steps before we can safely return to ring-3. This is not the same as longjmps
8462	* to ring-3, this is voluntary and prepares the guest so it may continue
8463	* executing outside HM (recompiler/IEM).
8464	*
8465	* @returns VBox status code.
8466	* @param pVCpu The cross context virtual CPU structure.
8467	* @param rcExit The reason for exiting to ring-3. Can be
8468	* VINF_VMM_UNKNOWN_RING3_CALL.
8469	*/
8470	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8471	{
8472	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8473
8474	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8475	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8476	{
8477	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8478	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8479	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8480	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8481	}
8482
8483	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8484	VMMRZCallRing3Disable(pVCpu);
8485	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8486
8487	/*
8488	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8489	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8490	*
8491	* This is because execution may continue from ring-3 and we would need to inject
8492	* the event from there (hence place it back in TRPM).
8493	*/
8494	if (pVCpu->hm.s.Event.fPending)
8495	{
8496	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8497	Assert(!pVCpu->hm.s.Event.fPending);
8498
8499	/* Clear the events from the VMCS. */
8500	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8501	AssertRCReturn(rc, rc);
8502	}
8503	#ifdef VBOX_STRICT
8504	else
8505	{
8506	/*
8507	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8508	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8509	* occasionally, see @bugref{9180#c42}.
8510	*/
8511	uint32_t uEntryIntInfo;
8512	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8513	AssertRC(rc);
8514	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8515	}
8516	#endif
8517
8518	/*
8519	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8520	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8521	* (e.g. TPR below threshold).
8522	*/
8523	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8524	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8525	AssertRCReturn(rc, rc);
8526
8527	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8528	and if we're injecting an event we should have a TRPM trap pending. */
8529	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8530	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8531	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8532	#endif
8533
8534	/* Save guest state and restore host state bits. */
8535	rc = hmR0VmxLeaveSession(pVCpu);
8536	AssertRCReturn(rc, rc);
8537	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8538
8539	/* Thread-context hooks are unregistered at this point!!! */
8540
8541	/* Sync recompiler state. */
8542	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8543	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8544	\| CPUM_CHANGED_LDTR
8545	\| CPUM_CHANGED_GDTR
8546	\| CPUM_CHANGED_IDTR
8547	\| CPUM_CHANGED_TR
8548	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8549	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8550	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8551	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8552
8553	Assert(!pVCpu->hm.s.fClearTrapFlag);
8554
8555	/* Update the exit-to-ring 3 reason. */
8556	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8557
8558	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8559	if ( rcExit != VINF_EM_RAW_INTERRUPT
8560	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8561	{
8562	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8563	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8564	}
8565
8566	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8567
8568	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8569	VMMRZCallRing3RemoveNotification(pVCpu);
8570	VMMRZCallRing3Enable(pVCpu);
8571
8572	return rc;
8573	}
8574
8575
8576	/**
8577	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8578	* longjump to ring-3 and possibly get preempted.
8579	*
8580	* @returns VBox status code.
8581	* @param pVCpu The cross context virtual CPU structure.
8582	* @param enmOperation The operation causing the ring-3 longjump.
8583	* @param pvUser User argument, currently unused, NULL.
8584	*/
8585	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8586	{
8587	RT_NOREF(pvUser);
8588	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8589	{
8590	/*
8591	* !!! IMPORTANT !!!
8592	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8593	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8594	*/
8595	VMMRZCallRing3RemoveNotification(pVCpu);
8596	VMMRZCallRing3Disable(pVCpu);
8597	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8598	RTThreadPreemptDisable(&PreemptState);
8599
8600	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8601	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8602	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8603	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8604
8605	#if HC_ARCH_BITS == 64
8606	/* Restore host-state bits that VT-x only restores partially. */
8607	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8608	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8609	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8610	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8611	#endif
8612
8613	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8614	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8615	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8616
8617	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8618	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8619	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8620
8621	/* Clear the current VMCS data back to memory. */
8622	hmR0VmxClearVmcs(pVmcsInfo);
8623
8624	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8625	VMMR0ThreadCtxHookDisable(pVCpu);
8626	HMR0LeaveCpu(pVCpu);
8627	RTThreadPreemptRestore(&PreemptState);
8628	return VINF_SUCCESS;
8629	}
8630
8631	Assert(pVCpu);
8632	Assert(pvUser);
8633	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8634	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8635
8636	VMMRZCallRing3Disable(pVCpu);
8637	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8638
8639	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8640
8641	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8642	AssertRCReturn(rc, rc);
8643
8644	VMMRZCallRing3Enable(pVCpu);
8645	return VINF_SUCCESS;
8646	}
8647
8648
8649	/**
8650	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8651	* stack.
8652	*
8653	* @returns Strict VBox status code (i.e. informational status codes too).
8654	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8655	* @param pVCpu The cross context virtual CPU structure.
8656	* @param uValue The value to push to the guest stack.
8657	*/
8658	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8659	{
8660	/*
8661	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8662	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8663	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8664	*/
8665	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8666	if (pCtx->sp == 1)
8667	return VINF_EM_RESET;
8668	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8669	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8670	AssertRC(rc);
8671	return rc;
8672	}
8673
8674
8675	/**
8676	* Injects an event into the guest upon VM-entry by updating the relevant fields
8677	* in the VM-entry area in the VMCS.
8678	*
8679	* @returns Strict VBox status code (i.e. informational status codes too).
8680	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8681	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8682	*
8683	* @param pVCpu The cross context virtual CPU structure.
8684	* @param pVmxTransient The VMX-transient structure.
8685	* @param pEvent The event being injected.
8686	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8687	* This will be updated if necessary. This cannot not
8688	* be NULL.
8689	* @param fStepping Whether we're single-stepping guest execution and
8690	* should return VINF_EM_DBG_STEPPED if the event is
8691	* injected directly (registers modified by us, not by
8692	* hardware on VM-entry).
8693	*/
8694	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8695	uint32_t *pfIntrState)
8696	{
8697	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8698	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8699	Assert(pfIntrState);
8700
8701	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8702	uint32_t u32IntInfo = pEvent->u64IntInfo;
8703	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8704	uint32_t const cbInstr = pEvent->cbInstr;
8705	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8706	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8707	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8708
8709	#ifdef VBOX_STRICT
8710	/*
8711	* Validate the error-code-valid bit for hardware exceptions.
8712	* No error codes for exceptions in real-mode.
8713	*
8714	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8715	*/
8716	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8717	&& !CPUMIsGuestInRealModeEx(pCtx))
8718	{
8719	switch (uVector)
8720	{
8721	case X86_XCPT_PF:
8722	case X86_XCPT_DF:
8723	case X86_XCPT_TS:
8724	case X86_XCPT_NP:
8725	case X86_XCPT_SS:
8726	case X86_XCPT_GP:
8727	case X86_XCPT_AC:
8728	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8729	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8730	RT_FALL_THRU();
8731	default:
8732	break;
8733	}
8734	}
8735
8736	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8737	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8738	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8739	#endif
8740
8741	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8742
8743	/*
8744	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8745	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8746	* interrupt handler in the (real-mode) guest.
8747	*
8748	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8749	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8750	*/
8751	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8752	{
8753	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8754	{
8755	/*
8756	* For CPUs with unrestricted guest execution enabled and with the guest
8757	* in real-mode, we must not set the deliver-error-code bit.
8758	*
8759	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8760	*/
8761	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8762	}
8763	else
8764	{
8765	PVM pVM = pVCpu->CTX_SUFF(pVM);
8766	Assert(PDMVmmDevHeapIsEnabled(pVM));
8767	Assert(pVM->hm.s.vmx.pRealModeTSS);
8768	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8769
8770	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8771	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8772	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8773	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8774	AssertRCReturn(rc2, rc2);
8775
8776	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8777	size_t const cbIdtEntry = sizeof(X86IDTR16);
8778	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8779	{
8780	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8781	if (uVector == X86_XCPT_DF)
8782	return VINF_EM_RESET;
8783
8784	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8785	No error codes for exceptions in real-mode. */
8786	if (uVector == X86_XCPT_GP)
8787	{
8788	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8789	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8790	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8791	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8792	HMEVENT EventXcptDf;
8793	RT_ZERO(EventXcptDf);
8794	EventXcptDf.u64IntInfo = uXcptDfInfo;
8795	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8796	}
8797
8798	/*
8799	* If we're injecting an event with no valid IDT entry, inject a #GP.
8800	* No error codes for exceptions in real-mode.
8801	*
8802	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8803	*/
8804	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8805	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8806	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8807	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8808	HMEVENT EventXcptGp;
8809	RT_ZERO(EventXcptGp);
8810	EventXcptGp.u64IntInfo = uXcptGpInfo;
8811	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8812	}
8813
8814	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8815	uint16_t uGuestIp = pCtx->ip;
8816	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8817	{
8818	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8819	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8820	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8821	}
8822	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8823	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8824
8825	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8826	X86IDTR16 IdtEntry;
8827	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8828	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8829	AssertRCReturn(rc2, rc2);
8830
8831	/* Construct the stack frame for the interrupt/exception handler. */
8832	VBOXSTRICTRC rcStrict;
8833	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8834	if (rcStrict == VINF_SUCCESS)
8835	{
8836	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8837	if (rcStrict == VINF_SUCCESS)
8838	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8839	}
8840
8841	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8842	if (rcStrict == VINF_SUCCESS)
8843	{
8844	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8845	pCtx->rip = IdtEntry.offSel;
8846	pCtx->cs.Sel = IdtEntry.uSel;
8847	pCtx->cs.ValidSel = IdtEntry.uSel;
8848	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8849	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8850	&& uVector == X86_XCPT_PF)
8851	pCtx->cr2 = GCPtrFault;
8852
8853	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8854	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8855	\| HM_CHANGED_GUEST_RSP);
8856
8857	/*
8858	* If we delivered a hardware exception (other than an NMI) and if there was
8859	* block-by-STI in effect, we should clear it.
8860	*/
8861	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8862	{
8863	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8864	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8865	Log4Func(("Clearing inhibition due to STI\n"));
8866	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8867	}
8868
8869	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8870	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8871
8872	/*
8873	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8874	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8875	*/
8876	pVCpu->hm.s.Event.fPending = false;
8877
8878	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8879	if (fStepping)
8880	rcStrict = VINF_EM_DBG_STEPPED;
8881	}
8882	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8883	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8884	return rcStrict;
8885	}
8886	}
8887
8888	/*
8889	* Validate.
8890	*/
8891	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8892	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8893
8894	/*
8895	* Inject the event into the VMCS.
8896	*/
8897	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8898	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8899	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8900	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8901	AssertRCReturn(rc, rc);
8902
8903	/*
8904	* Update guest CR2 if this is a page-fault.
8905	*/
8906	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8907	&& uVector == X86_XCPT_PF)
8908	pCtx->cr2 = GCPtrFault;
8909
8910	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8911	return VINF_SUCCESS;
8912	}
8913
8914
8915	/**
8916	* Evaluates the event to be delivered to the guest and sets it as the pending
8917	* event.
8918	*
8919	* @returns Strict VBox status code (i.e. informational status codes too).
8920	* @param pVCpu The cross context virtual CPU structure.
8921	* @param pVmxTransient The VMX-transient structure.
8922	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8923	*/
8924	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8925	{
8926	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8927	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8928
8929	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8930	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8931	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8932	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8933	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8934
8935	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8936	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8937	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8938	Assert(!TRPMHasTrap(pVCpu));
8939	Assert(pfIntrState);
8940
8941	*pfIntrState = fIntrState;
8942
8943	/*
8944	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8945	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8946	*/
8947	/** @todo SMI. SMIs take priority over NMIs. */
8948	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8949	{
8950	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8951	if ( !pVCpu->hm.s.Event.fPending
8952	&& !fBlockNmi
8953	&& !fBlockSti
8954	&& !fBlockMovSS)
8955	{
8956	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8957	if ( pVmxTransient->fIsNestedGuest
8958	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8959	return IEMExecVmxVmexitXcptNmi(pVCpu);
8960	#endif
8961	hmR0VmxSetPendingXcptNmi(pVCpu);
8962	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8963	Log4Func(("Pending NMI\n"));
8964	}
8965	else
8966	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8967	}
8968	/*
8969	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8970	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8971	*/
8972	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8973	&& !pVCpu->hm.s.fSingleInstruction)
8974	{
8975	Assert(!DBGFIsStepping(pVCpu));
8976	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8977	AssertRCReturn(rc, rc);
8978	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8979	if ( !pVCpu->hm.s.Event.fPending
8980	&& !fBlockInt
8981	&& !fBlockSti
8982	&& !fBlockMovSS)
8983	{
8984	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8985	if ( pVmxTransient->fIsNestedGuest
8986	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8987	{
8988	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8989	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8990	return rcStrict;
8991	}
8992	#endif
8993	uint8_t u8Interrupt;
8994	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8995	if (RT_SUCCESS(rc))
8996	{
8997	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8998	if ( pVmxTransient->fIsNestedGuest
8999	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
9000	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
9001	{
9002	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
9003	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9004	return rcStrict;
9005	}
9006	#endif
9007	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
9008	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
9009	}
9010	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
9011	{
9012	if ( !pVmxTransient->fIsNestedGuest
9013	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
9014	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
9015	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
9016
9017	/*
9018	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
9019	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
9020	* need to re-set this force-flag here.
9021	*/
9022	}
9023	else
9024	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
9025	}
9026	else
9027	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
9028	}
9029
9030	return VINF_SUCCESS;
9031	}
9032
9033
9034	/**
9035	* Injects any pending events into the guest if the guest is in a state to
9036	* receive them.
9037	*
9038	* @returns Strict VBox status code (i.e. informational status codes too).
9039	* @param pVCpu The cross context virtual CPU structure.
9040	* @param pVmxTransient The VMX-transient structure.
9041	* @param fIntrState The VT-x guest-interruptibility state.
9042	* @param fStepping Whether we are single-stepping the guest using the
9043	* hypervisor debugger and should return
9044	* VINF_EM_DBG_STEPPED if the event was dispatched
9045	* directly.
9046	*/
9047	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9048	{
9049	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9050	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9051
9052	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9053	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9054
9055	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9056	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9057	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9058	Assert(!TRPMHasTrap(pVCpu));
9059
9060	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9061	if (pVCpu->hm.s.Event.fPending)
9062	{
9063	/*
9064	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9065	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9066	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9067	*
9068	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9069	*/
9070	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9071	#ifdef VBOX_STRICT
9072	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9073	{
9074	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9075	Assert(!fBlockInt);
9076	Assert(!fBlockSti);
9077	Assert(!fBlockMovSS);
9078	}
9079	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9080	{
9081	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9082	Assert(!fBlockSti);
9083	Assert(!fBlockMovSS);
9084	Assert(!fBlockNmi);
9085	}
9086	#endif
9087	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9088	uIntType));
9089
9090	/*
9091	* Inject the event and get any changes to the guest-interruptibility state.
9092	*
9093	* The guest-interruptibility state may need to be updated if we inject the event
9094	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9095	*/
9096	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9097	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9098
9099	/*
9100	* If we are executing a nested-guest make sure that we should intercept subsequent
9101	* events. The one we are injecting might be part of VM-entry.
9102	*/
9103	if (pVmxTransient->fIsNestedGuest)
9104	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
9105
9106	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9107	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9108	else
9109	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9110	}
9111
9112	/*
9113	* Update the guest-interruptibility state.
9114	*
9115	* This is required for the real-on-v86 software interrupt injection case above, as well as
9116	* updates to the guest state from ring-3 or IEM/REM.
9117	*/
9118	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9119	AssertRCReturn(rc, rc);
9120
9121	/*
9122	* There's no need to clear the VM-entry interruption-information field here if we're not
9123	* injecting anything. VT-x clears the valid bit on every VM-exit.
9124	*
9125	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9126	*/
9127
9128	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9129	NOREF(fBlockMovSS); NOREF(fBlockSti);
9130	return rcStrict;
9131	}
9132
9133
9134	/**
9135	* Enters the VT-x session.
9136	*
9137	* @returns VBox status code.
9138	* @param pVCpu The cross context virtual CPU structure.
9139	*/
9140	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9141	{
9142	AssertPtr(pVCpu);
9143	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9144	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9145
9146	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9147	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9148	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9149
9150	#ifdef VBOX_STRICT
9151	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9152	RTCCUINTREG uHostCr4 = ASMGetCR4();
9153	if (!(uHostCr4 & X86_CR4_VMXE))
9154	{
9155	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9156	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9157	}
9158	#endif
9159
9160	/*
9161	* Load the appropriate VMCS as the current and active one.
9162	*/
9163	PVMXVMCSINFO pVmcsInfo;
9164	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9165	if (!fInNestedGuestMode)
9166	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9167	else
9168	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9169	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9170	if (RT_SUCCESS(rc))
9171	{
9172	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9173	pVCpu->hm.s.fLeaveDone = false;
9174	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9175
9176	/*
9177	* Do the EMT scheduled L1D flush here if needed.
9178	*/
9179	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9180	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9181	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9182	hmR0MdsClear();
9183	}
9184	return rc;
9185	}
9186
9187
9188	/**
9189	* The thread-context callback (only on platforms which support it).
9190	*
9191	* @param enmEvent The thread-context event.
9192	* @param pVCpu The cross context virtual CPU structure.
9193	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9194	* @thread EMT(pVCpu)
9195	*/
9196	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9197	{
9198	AssertPtr(pVCpu);
9199	RT_NOREF1(fGlobalInit);
9200
9201	switch (enmEvent)
9202	{
9203	case RTTHREADCTXEVENT_OUT:
9204	{
9205	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9206	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9207	VMCPU_ASSERT_EMT(pVCpu);
9208
9209	/* No longjmps (logger flushes, locks) in this fragile context. */
9210	VMMRZCallRing3Disable(pVCpu);
9211	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9212
9213	/* Restore host-state (FPU, debug etc.) */
9214	if (!pVCpu->hm.s.fLeaveDone)
9215	{
9216	/*
9217	* Do -not- import the guest-state here as we might already be in the middle of importing
9218	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9219	*/
9220	hmR0VmxLeave(pVCpu, false /* fImportState */);
9221	pVCpu->hm.s.fLeaveDone = true;
9222	}
9223
9224	/* Leave HM context, takes care of local init (term). */
9225	int rc = HMR0LeaveCpu(pVCpu);
9226	AssertRC(rc);
9227
9228	/* Restore longjmp state. */
9229	VMMRZCallRing3Enable(pVCpu);
9230	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9231	break;
9232	}
9233
9234	case RTTHREADCTXEVENT_IN:
9235	{
9236	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9237	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9238	VMCPU_ASSERT_EMT(pVCpu);
9239
9240	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9241	VMMRZCallRing3Disable(pVCpu);
9242	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9243
9244	/* Initialize the bare minimum state required for HM. This takes care of
9245	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9246	int rc = hmR0EnterCpu(pVCpu);
9247	AssertRC(rc);
9248	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9249	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9250
9251	/* Load the active VMCS as the current one. */
9252	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9253	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9254	AssertRC(rc);
9255	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9256	pVCpu->hm.s.fLeaveDone = false;
9257
9258	/* Do the EMT scheduled L1D flush if needed. */
9259	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9260	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9261
9262	/* Restore longjmp state. */
9263	VMMRZCallRing3Enable(pVCpu);
9264	break;
9265	}
9266
9267	default:
9268	break;
9269	}
9270	}
9271
9272
9273	/**
9274	* Exports the host state into the VMCS host-state area.
9275	* Sets up the VM-exit MSR-load area.
9276	*
9277	* The CPU state will be loaded from these fields on every successful VM-exit.
9278	*
9279	* @returns VBox status code.
9280	* @param pVCpu The cross context virtual CPU structure.
9281	*
9282	* @remarks No-long-jump zone!!!
9283	*/
9284	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9285	{
9286	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9287
9288	int rc = VINF_SUCCESS;
9289	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9290	{
9291	rc = hmR0VmxExportHostControlRegs();
9292	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9293
9294	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9295	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9296
9297	rc = hmR0VmxExportHostMsrs(pVCpu);
9298	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9299
9300	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9301	}
9302	return rc;
9303	}
9304
9305
9306	/**
9307	* Saves the host state in the VMCS host-state.
9308	*
9309	* @returns VBox status code.
9310	* @param pVCpu The cross context virtual CPU structure.
9311	*
9312	* @remarks No-long-jump zone!!!
9313	*/
9314	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9315	{
9316	AssertPtr(pVCpu);
9317	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9318
9319	/*
9320	* Export the host state here while entering HM context.
9321	* When thread-context hooks are used, we might get preempted and have to re-save the host
9322	* state but most of the time we won't be, so do it here before we disable interrupts.
9323	*/
9324	return hmR0VmxExportHostState(pVCpu);
9325	}
9326
9327
9328	/**
9329	* Exports the guest state into the VMCS guest-state area.
9330	*
9331	* The will typically be done before VM-entry when the guest-CPU state and the
9332	* VMCS state may potentially be out of sync.
9333	*
9334	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9335	* VM-entry controls.
9336	* Sets up the appropriate VMX non-root function to execute guest code based on
9337	* the guest CPU mode.
9338	*
9339	* @returns VBox strict status code.
9340	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9341	* without unrestricted guest execution and the VMMDev is not presently
9342	* mapped (e.g. EFI32).
9343	*
9344	* @param pVCpu The cross context virtual CPU structure.
9345	* @param pVmxTransient The VMX-transient structure.
9346	*
9347	* @remarks No-long-jump zone!!!
9348	*/
9349	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9350	{
9351	AssertPtr(pVCpu);
9352	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9353	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9354
9355	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9356
9357	/*
9358	* Determine real-on-v86 mode.
9359	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9360	*/
9361	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9362	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9363	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9364	pVmcsInfo->RealMode. fRealOnV86Active = false;
9365	else
9366	{
9367	Assert(!pVmxTransient->fIsNestedGuest);
9368	pVmcsInfo->RealMode.fRealOnV86Active = true;
9369	}
9370
9371	/*
9372	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9373	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9374	*/
9375	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9376	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9377	* be a need to evaluate this everytime since I'm pretty sure we intercept
9378	* all guest paging mode changes. */
9379	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9380	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9381
9382	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9383	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9384
9385	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9386	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9387
9388	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9389	if (rcStrict == VINF_SUCCESS)
9390	{ /* likely */ }
9391	else
9392	{
9393	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9394	return rcStrict;
9395	}
9396
9397	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9398	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9399
9400	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9401	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9402
9403	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9404	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9405
9406	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9407	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9408
9409	rc = hmR0VmxExportGuestRip(pVCpu);
9410	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9411	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9412	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9413
9414	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9415	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9416	\| HM_CHANGED_GUEST_CR2
9417	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9418	\| HM_CHANGED_GUEST_X87
9419	\| HM_CHANGED_GUEST_SSE_AVX
9420	\| HM_CHANGED_GUEST_OTHER_XSAVE
9421	\| HM_CHANGED_GUEST_XCRx
9422	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9423	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9424	\| HM_CHANGED_GUEST_TSC_AUX
9425	\| HM_CHANGED_GUEST_OTHER_MSRS
9426	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9427	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9428
9429	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9430	return rc;
9431	}
9432
9433
9434	/**
9435	* Exports the state shared between the host and guest into the VMCS.
9436	*
9437	* @param pVCpu The cross context virtual CPU structure.
9438	* @param pVmxTransient The VMX-transient structure.
9439	*
9440	* @remarks No-long-jump zone!!!
9441	*/
9442	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9443	{
9444	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9445	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9446
9447	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9448	{
9449	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9450	AssertRC(rc);
9451	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9452
9453	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9454	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9455	{
9456	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9457	AssertRC(rc);
9458	}
9459	}
9460
9461	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9462	{
9463	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9464	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9465	}
9466
9467	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9468	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9469	}
9470
9471
9472	/**
9473	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9474	*
9475	* @returns Strict VBox status code (i.e. informational status codes too).
9476	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9477	* without unrestricted guest execution and the VMMDev is not presently
9478	* mapped (e.g. EFI32).
9479	*
9480	* @param pVCpu The cross context virtual CPU structure.
9481	* @param pVmxTransient The VMX-transient structure.
9482	*
9483	* @remarks No-long-jump zone!!!
9484	*/
9485	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9486	{
9487	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9488	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9489	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9490
9491	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9492	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9493	#endif
9494
9495	/*
9496	* For many exits it's only RIP that changes and hence try to export it first
9497	* without going through a lot of change flag checks.
9498	*/
9499	VBOXSTRICTRC rcStrict;
9500	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9501	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9502	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9503	{
9504	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9505	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9506	{ /* likely */}
9507	else
9508	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9509	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9510	}
9511	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9512	{
9513	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9514	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9515	{ /* likely */}
9516	else
9517	{
9518	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9519	VBOXSTRICTRC_VAL(rcStrict)));
9520	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9521	return rcStrict;
9522	}
9523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9524	}
9525	else
9526	rcStrict = VINF_SUCCESS;
9527
9528	#ifdef VBOX_STRICT
9529	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9530	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9531	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9532	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9533	("fCtxChanged=%#RX64\n", fCtxChanged));
9534	#endif
9535	return rcStrict;
9536	}
9537
9538
9539	/**
9540	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9541	* and update error record fields accordingly.
9542	*
9543	* @returns VMX_IGS_* error codes.
9544	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9545	* wrong with the guest state.
9546	*
9547	* @param pVCpu The cross context virtual CPU structure.
9548	* @param pVmcsInfo The VMCS info. object.
9549	*
9550	* @remarks This function assumes our cache of the VMCS controls
9551	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9552	*/
9553	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9554	{
9555	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9556	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9557	uError = (err); \
9558	break; \
9559	} else do { } while (0)
9560
9561	int rc;
9562	PVM pVM = pVCpu->CTX_SUFF(pVM);
9563	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9564	uint32_t uError = VMX_IGS_ERROR;
9565	uint32_t u32Val;
9566	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9567
9568	do
9569	{
9570	/*
9571	* CR0.
9572	*/
9573	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9574	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9575	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9576	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9577	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9578	if (fUnrestrictedGuest)
9579	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9580
9581	uint32_t u32GuestCr0;
9582	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9583	AssertRCBreak(rc);
9584	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9585	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9586	if ( !fUnrestrictedGuest
9587	&& (u32GuestCr0 & X86_CR0_PG)
9588	&& !(u32GuestCr0 & X86_CR0_PE))
9589	{
9590	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9591	}
9592
9593	/*
9594	* CR4.
9595	*/
9596	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9597	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9598	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9599
9600	uint32_t u32GuestCr4;
9601	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9602	AssertRCBreak(rc);
9603	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9604	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9605
9606	/*
9607	* IA32_DEBUGCTL MSR.
9608	*/
9609	uint64_t u64Val;
9610	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9611	AssertRCBreak(rc);
9612	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9613	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9614	{
9615	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9616	}
9617	uint64_t u64DebugCtlMsr = u64Val;
9618
9619	#ifdef VBOX_STRICT
9620	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9621	AssertRCBreak(rc);
9622	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9623	#endif
9624	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9625
9626	/*
9627	* RIP and RFLAGS.
9628	*/
9629	uint32_t u32Eflags;
9630	#if HC_ARCH_BITS == 64
9631	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9632	AssertRCBreak(rc);
9633	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9634	if ( !fLongModeGuest
9635	\|\| !pCtx->cs.Attr.n.u1Long)
9636	{
9637	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9638	}
9639	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9640	* must be identical if the "IA-32e mode guest" VM-entry
9641	* control is 1 and CS.L is 1. No check applies if the
9642	* CPU supports 64 linear-address bits. */
9643
9644	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9645	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9646	AssertRCBreak(rc);
9647	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9648	VMX_IGS_RFLAGS_RESERVED);
9649	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9650	u32Eflags = u64Val;
9651	#else
9652	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9653	AssertRCBreak(rc);
9654	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9655	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9656	#endif
9657
9658	if ( fLongModeGuest
9659	\|\| ( fUnrestrictedGuest
9660	&& !(u32GuestCr0 & X86_CR0_PE)))
9661	{
9662	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9663	}
9664
9665	uint32_t u32EntryInfo;
9666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9667	AssertRCBreak(rc);
9668	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9669	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9670	{
9671	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9672	}
9673
9674	/*
9675	* 64-bit checks.
9676	*/
9677	#if HC_ARCH_BITS == 64
9678	if (fLongModeGuest)
9679	{
9680	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9681	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9682	}
9683
9684	if ( !fLongModeGuest
9685	&& (u32GuestCr4 & X86_CR4_PCIDE))
9686	{
9687	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9688	}
9689
9690	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9691	* 51:32 beyond the processor's physical-address width are 0. */
9692
9693	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9694	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9695	{
9696	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9697	}
9698
9699	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9700	AssertRCBreak(rc);
9701	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9702
9703	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9704	AssertRCBreak(rc);
9705	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9706	#endif
9707
9708	/*
9709	* PERF_GLOBAL MSR.
9710	*/
9711	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9712	{
9713	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9714	AssertRCBreak(rc);
9715	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9716	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9717	}
9718
9719	/*
9720	* PAT MSR.
9721	*/
9722	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9723	{
9724	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9725	AssertRCBreak(rc);
9726	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9727	for (unsigned i = 0; i < 8; i++)
9728	{
9729	uint8_t u8Val = (u64Val & 0xff);
9730	if ( u8Val != 0 /* UC */
9731	&& u8Val != 1 /* WC */
9732	&& u8Val != 4 /* WT */
9733	&& u8Val != 5 /* WP */
9734	&& u8Val != 6 /* WB */
9735	&& u8Val != 7 /* UC- */)
9736	{
9737	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9738	}
9739	u64Val >>= 8;
9740	}
9741	}
9742
9743	/*
9744	* EFER MSR.
9745	*/
9746	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9747	{
9748	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9749	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9750	AssertRCBreak(rc);
9751	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9752	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9753	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9754	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9755	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9756	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9757	* iemVmxVmentryCheckGuestState(). */
9758	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9759	\|\| !(u32GuestCr0 & X86_CR0_PG)
9760	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9761	VMX_IGS_EFER_LMA_LME_MISMATCH);
9762	}
9763
9764	/*
9765	* Segment registers.
9766	*/
9767	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9768	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9769	if (!(u32Eflags & X86_EFL_VM))
9770	{
9771	/* CS */
9772	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9773	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9774	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9775	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9776	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9777	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9778	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9779	/* CS cannot be loaded with NULL in protected mode. */
9780	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9781	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9782	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9783	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9784	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9785	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9786	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9787	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9788	else
9789	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9790
9791	/* SS */
9792	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9793	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9794	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9795	if ( !(pCtx->cr0 & X86_CR0_PE)
9796	\|\| pCtx->cs.Attr.n.u4Type == 3)
9797	{
9798	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9799	}
9800	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9801	{
9802	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9803	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9804	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9805	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9806	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9807	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9808	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9809	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9810	}
9811
9812	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9813	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9814	{
9815	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9816	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9817	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9818	\|\| pCtx->ds.Attr.n.u4Type > 11
9819	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9820	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9821	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9822	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9823	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9824	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9825	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9826	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9827	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9828	}
9829	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9830	{
9831	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9832	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9833	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9834	\|\| pCtx->es.Attr.n.u4Type > 11
9835	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9836	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9837	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9838	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9839	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9840	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9841	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9842	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9843	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9844	}
9845	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9846	{
9847	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9848	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9849	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9850	\|\| pCtx->fs.Attr.n.u4Type > 11
9851	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9852	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9853	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9854	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9855	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9856	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9857	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9858	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9859	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9860	}
9861	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9862	{
9863	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9864	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9865	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9866	\|\| pCtx->gs.Attr.n.u4Type > 11
9867	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9868	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9869	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9870	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9871	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9872	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9873	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9874	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9875	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9876	}
9877	/* 64-bit capable CPUs. */
9878	#if HC_ARCH_BITS == 64
9879	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9880	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9881	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9882	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9883	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9884	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9885	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9886	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9887	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9888	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9889	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9890	#endif
9891	}
9892	else
9893	{
9894	/* V86 mode checks. */
9895	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9896	if (pVmcsInfo->RealMode.fRealOnV86Active)
9897	{
9898	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9899	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9900	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9901	}
9902	else
9903	{
9904	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9905	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9906	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9907	}
9908
9909	/* CS */
9910	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9911	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9912	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9913	/* SS */
9914	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9915	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9916	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9917	/* DS */
9918	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9919	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9920	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9921	/* ES */
9922	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9923	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9924	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9925	/* FS */
9926	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9927	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9928	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9929	/* GS */
9930	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9931	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9932	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9933	/* 64-bit capable CPUs. */
9934	#if HC_ARCH_BITS == 64
9935	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9936	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9937	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9938	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9939	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9940	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9941	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9942	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9943	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9944	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9945	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9946	#endif
9947	}
9948
9949	/*
9950	* TR.
9951	*/
9952	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9953	/* 64-bit capable CPUs. */
9954	#if HC_ARCH_BITS == 64
9955	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9956	#endif
9957	if (fLongModeGuest)
9958	{
9959	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9960	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9961	}
9962	else
9963	{
9964	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9965	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9966	VMX_IGS_TR_ATTR_TYPE_INVALID);
9967	}
9968	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9969	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9970	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9971	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9972	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9973	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9974	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9975	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9976
9977	/*
9978	* GDTR and IDTR.
9979	*/
9980	#if HC_ARCH_BITS == 64
9981	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9982	AssertRCBreak(rc);
9983	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9984
9985	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9986	AssertRCBreak(rc);
9987	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9988	#endif
9989
9990	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9991	AssertRCBreak(rc);
9992	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9993
9994	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9995	AssertRCBreak(rc);
9996	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9997
9998	/*
9999	* Guest Non-Register State.
10000	*/
10001	/* Activity State. */
10002	uint32_t u32ActivityState;
10003	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10004	AssertRCBreak(rc);
10005	HMVMX_CHECK_BREAK( !u32ActivityState
10006	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10007	VMX_IGS_ACTIVITY_STATE_INVALID);
10008	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10009	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10010	uint32_t u32IntrState;
10011	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10012	AssertRCBreak(rc);
10013	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10014	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10015	{
10016	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10017	}
10018
10019	/** @todo Activity state and injecting interrupts. Left as a todo since we
10020	* currently don't use activity states but ACTIVE. */
10021
10022	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10023	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10024
10025	/* Guest interruptibility-state. */
10026	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10027	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10028	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10029	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10030	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10031	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10032	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10033	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10034	{
10035	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10036	{
10037	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10038	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10039	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10040	}
10041	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10042	{
10043	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10044	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10045	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10046	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10047	}
10048	}
10049	/** @todo Assumes the processor is not in SMM. */
10050	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10051	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10052	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10053	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10054	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10055	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10056	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10057	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10058	{
10059	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10060	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10061	}
10062
10063	/* Pending debug exceptions. */
10064	#if HC_ARCH_BITS == 64
10065	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10066	AssertRCBreak(rc);
10067	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10068	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10069	u32Val = u64Val; /* For pending debug exceptions checks below. */
10070	#else
10071	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10072	AssertRCBreak(rc);
10073	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10074	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10075	#endif
10076
10077	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10078	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10079	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10080	{
10081	if ( (u32Eflags & X86_EFL_TF)
10082	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10083	{
10084	/* Bit 14 is PendingDebug.BS. */
10085	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10086	}
10087	if ( !(u32Eflags & X86_EFL_TF)
10088	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10089	{
10090	/* Bit 14 is PendingDebug.BS. */
10091	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10092	}
10093	}
10094
10095	/* VMCS link pointer. */
10096	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10097	AssertRCBreak(rc);
10098	if (u64Val != UINT64_C(0xffffffffffffffff))
10099	{
10100	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10101	/** @todo Bits beyond the processor's physical-address width MBZ. */
10102	/** @todo 32-bit located in memory referenced by value of this field (as a
10103	* physical address) must contain the processor's VMCS revision ID. */
10104	/** @todo SMM checks. */
10105	}
10106
10107	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10108	* not using nested paging? */
10109	if ( pVM->hm.s.fNestedPaging
10110	&& !fLongModeGuest
10111	&& CPUMIsGuestInPAEModeEx(pCtx))
10112	{
10113	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10114	AssertRCBreak(rc);
10115	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10116
10117	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10118	AssertRCBreak(rc);
10119	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10120
10121	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10122	AssertRCBreak(rc);
10123	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10124
10125	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10126	AssertRCBreak(rc);
10127	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10128	}
10129
10130	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10131	if (uError == VMX_IGS_ERROR)
10132	uError = VMX_IGS_REASON_NOT_FOUND;
10133	} while (0);
10134
10135	pVCpu->hm.s.u32HMError = uError;
10136	return uError;
10137
10138	#undef HMVMX_ERROR_BREAK
10139	#undef HMVMX_CHECK_BREAK
10140	}
10141
10142
10143	/**
10144	* Setup the APIC-access page for virtualizing APIC access.
10145	*
10146	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
10147	* this not done as part of exporting guest state, see @bugref{8721}.
10148	*
10149	* @returns VBox status code.
10150	* @param pVCpu The cross context virtual CPU structure.
10151	*/
10152	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10153	{
10154	PVM pVM = pVCpu->CTX_SUFF(pVM);
10155	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10156
10157	Assert(PDMHasApic(pVM));
10158	Assert(u64MsrApicBase);
10159
10160	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10161	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10162
10163	/* Unalias any existing mapping. */
10164	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10165	AssertRCReturn(rc, rc);
10166
10167	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10168	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10169	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10170	AssertRCReturn(rc, rc);
10171
10172	/* Update the per-VCPU cache of the APIC base MSR. */
10173	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10174	return VINF_SUCCESS;
10175	}
10176
10177
10178	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10179	/**
10180	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10181	* nested-guest using hardware-assisted VMX.
10182	*
10183	* @param pVCpu The cross context virtual CPU structure.
10184	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10185	* @param pVmcsInfoGst The guest VMCS info. object.
10186	*/
10187	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10188	{
10189	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10190	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10191	Assert(pu64MsrBitmap);
10192
10193	/*
10194	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10195	* MSR that is intercepted by the guest is also intercepted while executing the
10196	* nested-guest using hardware-assisted VMX.
10197	*
10198	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10199	* nested-guest VM-exit even if the outer guest is not intercepting some
10200	* MSRs. We cannot assume the caller has initialized the nested-guest
10201	* MSR bitmap in this case.
10202	*
10203	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10204	* each VM-entry, hence initializing it once per-VM while setting up the
10205	* nested-guest VMCS is not sufficient.
10206	*/
10207	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10208	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10209	{
10210	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10211	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10212	Assert(pu64MsrBitmapNstGst);
10213	Assert(pu64MsrBitmapGst);
10214
10215	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10216	for (uint32_t i = 0; i < cFrags; i++)
10217	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10218	}
10219	else
10220	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10221	}
10222
10223
10224	/**
10225	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10226	* hardware-assisted VMX execution of the nested-guest.
10227	*
10228	* For a guest, we don't modify these controls once we set up the VMCS and hence
10229	* this function is never called.
10230	*
10231	* For nested-guests since the guest hypervisor provides these controls on every
10232	* nested-guest VM-entry and could potentially change them everytime we need to
10233	* merge them before every nested-guest VM-entry.
10234	*
10235	* @returns VBox status code.
10236	* @param pVCpu The cross context virtual CPU structure.
10237	*/
10238	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10239	{
10240	PVM pVM = pVCpu->CTX_SUFF(pVM);
10241	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10242	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10243	Assert(pVmcsNstGst);
10244
10245	/*
10246	* Merge the controls with the requirements of the guest VMCS.
10247	*
10248	* We do not need to validate the nested-guest VMX features specified in the
10249	* nested-guest VMCS with the features supported by the physical CPU as it's
10250	* already done by the VMLAUNCH/VMRESUME instruction emulation.
10251	*
10252	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
10253	* guest are derived from the VMX features supported by the physical CPU.
10254	*/
10255
10256	/* Pin-based VM-execution controls. */
10257	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10258
10259	/* Processor-based VM-execution controls. */
10260	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10261	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10262	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10263	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10264	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10265
10266	/* Secondary processor-based VM-execution controls. */
10267	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10268	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10269	\| VMX_PROC_CTLS2_INVPCID
10270	\| VMX_PROC_CTLS2_RDTSCP
10271	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10272	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10273	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10274	\| VMX_PROC_CTLS2_VMFUNC));
10275
10276	/*
10277	* VM-entry controls:
10278	* These controls contains state that depends on the nested-guest state (primarily
10279	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10280	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10281	* properly continue executing the nested-guest if the EFER MSR changes but does not
10282	* cause a nested-guest VM-exits.
10283	*
10284	* VM-exit controls:
10285	* These controls specify the host state on return. We cannot use the controls from
10286	* the guest hypervisor state as is as it would contain the guest state rather than
10287	* the host state. Since the host state is subject to change (e.g. preemption, trips
10288	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10289	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10290	*
10291	* VM-entry MSR-load:
10292	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10293	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10294	*
10295	* VM-exit MSR-store:
10296	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10297	* context back into the VM-exit MSR-store area.
10298	*
10299	* VM-exit MSR-load areas:
10300	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10301	* we can entirely ignore what the guest hypervisor wants to load here.
10302	*/
10303
10304	/*
10305	* Exception bitmap.
10306	*
10307	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10308	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10309	* keep the code more flexible if intercepting exceptions become more dynamic in
10310	* the future we do it as part of exporting the nested-guest state.
10311	*/
10312	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10313
10314	/*
10315	* CR0/CR4 guest/host mask.
10316	*
10317	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10318	* must cause VM-exits, so we need to merge them here.
10319	*/
10320	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10321	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10322
10323	/*
10324	* Page-fault error-code mask and match.
10325	*
10326	* Although we require unrestricted guest execution (and thereby nested-paging) for
10327	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10328	* normally intercept #PFs, it might intercept them for debugging purposes.
10329	*
10330	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10331	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10332	*/
10333	uint32_t u32XcptPFMask;
10334	uint32_t u32XcptPFMatch;
10335	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10336	{
10337	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10338	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10339	}
10340	else
10341	{
10342	u32XcptPFMask = 0;
10343	u32XcptPFMatch = 0;
10344	}
10345
10346	/*
10347	* Pause-Loop exiting.
10348	*/
10349	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10350	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10351
10352	/*
10353	* I/O Bitmap.
10354	*
10355	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10356	* always intercept all I/O port accesses.
10357	*/
10358	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10359
10360	/*
10361	* APIC-access page.
10362	*
10363	* The APIC-access page address has already been initialized while setting up the
10364	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10365	* should not be on any consequence to the host or to the guest for that matter, but
10366	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10367	* emulation to keep it simple.
10368	*/
10369
10370	/*
10371	* Virtual-APIC page and TPR threshold.
10372	*
10373	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10374	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10375	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10376	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10377	*/
10378	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10379	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10380	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10381	{
10382	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10383
10384	/*
10385	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10386	* we would fail to obtain a valid host-physical address for its guest-physical
10387	* address.
10388	*
10389	* We currently do not support this scenario. Maybe in the future if there is a
10390	* pressing need we can explore making this particular set of conditions work.
10391	* Right now we just cause a VM-entry failure.
10392	*
10393	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10394	* so should not really failure at the moment.
10395	*/
10396	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10397	}
10398	else
10399	{
10400	/*
10401	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10402	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10403	* be taken care of by EPT/shadow paging.
10404	*/
10405	if (pVM->hm.s.fAllow64BitGuests)
10406	{
10407	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10408	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10409	}
10410	}
10411
10412	/*
10413	* Validate basic assumptions.
10414	*/
10415	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10416	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10417	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10418	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10419
10420	/*
10421	* Commit it to the nested-guest VMCS.
10422	*/
10423	int rc = VINF_SUCCESS;
10424	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10425	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10426	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10427	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10428	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10429	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10430	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10431	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10432	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10433	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10434	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10435	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10436	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10437	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10438	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10439	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10440	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10441	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10442	{
10443	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10444	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10445	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10446	}
10447	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10448	{
10449	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10450	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10451	}
10452	AssertRCReturn(rc, rc);
10453
10454	/*
10455	* Update the nested-guest VMCS cache.
10456	*/
10457	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10458	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10459	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10460	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10461	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10462	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10463	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10464	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10465	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10466
10467	/*
10468	* MSR bitmap.
10469	*
10470	* The MSR bitmap address has already been initialized while setting up the
10471	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10472	*/
10473	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10474	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10475
10476	return VINF_SUCCESS;
10477	}
10478	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10479
10480
10481	/**
10482	* Does the preparations before executing guest code in VT-x.
10483	*
10484	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10485	* recompiler/IEM. We must be cautious what we do here regarding committing
10486	* guest-state information into the VMCS assuming we assuredly execute the
10487	* guest in VT-x mode.
10488	*
10489	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10490	* the common-state (TRPM/forceflags), we must undo those changes so that the
10491	* recompiler/IEM can (and should) use them when it resumes guest execution.
10492	* Otherwise such operations must be done when we can no longer exit to ring-3.
10493	*
10494	* @returns Strict VBox status code (i.e. informational status codes too).
10495	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10496	* have been disabled.
10497	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10498	* pending events).
10499	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10500	* double-fault into the guest.
10501	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10502	* dispatched directly.
10503	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10504	*
10505	* @param pVCpu The cross context virtual CPU structure.
10506	* @param pVmxTransient The VMX-transient structure.
10507	* @param fStepping Whether we are single-stepping the guest in the
10508	* hypervisor debugger. Makes us ignore some of the reasons
10509	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10510	* if event dispatching took place.
10511	*/
10512	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10513	{
10514	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10515
10516	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10517	if (pVmxTransient->fIsNestedGuest)
10518	{
10519	RT_NOREF2(pVCpu, fStepping);
10520	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10521	return VINF_EM_RESCHEDULE_REM;
10522	}
10523	#endif
10524
10525	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10526	PGMRZDynMapFlushAutoSet(pVCpu);
10527	#endif
10528
10529	/*
10530	* Check and process force flag actions, some of which might require us to go back to ring-3.
10531	*/
10532	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10533	if (rcStrict == VINF_SUCCESS)
10534	{ /* FFs don't get set all the time. */ }
10535	else
10536	return rcStrict;
10537
10538	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10539	/*
10540	* Switch to the nested-guest VMCS as we may have transitioned into executing
10541	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10542	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10543	*
10544	* We do this as late as possible to minimize (though not completely remove)
10545	* clearing/loading VMCS again due to premature trips to ring-3 above.
10546	*/
10547	if (pVmxTransient->fIsNestedGuest)
10548	{
10549	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10550	{
10551	/*
10552	* Ensure we have synced everything from the guest VMCS and also flag that
10553	* that we need to export the full (nested) guest-CPU context to the
10554	* nested-guest VMCS.
10555	*/
10556	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10557	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10558
10559	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10560	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10561	if (RT_LIKELY(rc == VINF_SUCCESS))
10562	{
10563	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10564	ASMSetFlags(fEFlags);
10565	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10566
10567	/*
10568	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10569	* flag that we need to update the host MSR values there. Even if we decide
10570	* in the future to share the VM-exit MSR-store area page with the guest,
10571	* if its content differs, we would have to update the host MSRs anyway.
10572	*/
10573	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10574	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10575	}
10576	else
10577	{
10578	ASMSetFlags(fEFlags);
10579	return rc;
10580	}
10581	}
10582
10583	/*
10584	* Merge guest VMCS controls with the nested-guest VMCS controls.
10585	*
10586	* Even if we have not executed the guest prior to this (e.g. when resuming
10587	* from a saved state), we should be okay with merging controls as we
10588	* initialize the guest VMCS controls as part of VM setup phase.
10589	*/
10590	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10591	{
10592	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10593	AssertRCReturn(rc, rc);
10594	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10595	}
10596	}
10597	#endif
10598
10599	/*
10600	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10601	*/
10602	PVM pVM = pVCpu->CTX_SUFF(pVM);
10603	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10604	&& hmR0VmxIsProcCtls2Set(pVCpu, pVmxTransient, VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10605	&& PDMHasApic(pVM))
10606	{
10607	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10608	AssertRCReturn(rc, rc);
10609	}
10610
10611	/*
10612	* Evaluate events to be injected into the guest.
10613	*
10614	* Events in TRPM can be injected without inspecting the guest state.
10615	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10616	* guest to cause a VM-exit the next time they are ready to receive the event.
10617	*/
10618	if (TRPMHasTrap(pVCpu))
10619	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10620
10621	uint32_t fIntrState;
10622	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10623
10624	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10625	/*
10626	* While evaluating pending events if something failed (unlikely) or if we were
10627	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10628	*/
10629	if (rcStrict != VINF_SUCCESS)
10630	return rcStrict;
10631	if ( pVmxTransient->fIsNestedGuest
10632	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10633	return VINF_VMX_VMEXIT;
10634	#else
10635	Assert(rcStrict == VINF_SUCCESS);
10636	#endif
10637
10638	/*
10639	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10640	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10641	* also result in triple-faulting the VM.
10642	*
10643	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10644	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10645	*/
10646	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10647	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10648	{ /* likely */ }
10649	else
10650	{
10651	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10652	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10653	return rcStrict;
10654	}
10655
10656	/*
10657	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10658	* import CR3 themselves. We will need to update them here, as even as late as the above
10659	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10660	* the below force flags to be set.
10661	*/
10662	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10663	{
10664	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10665	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10666	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10667	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10668	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10669	}
10670	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10671	{
10672	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10673	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10674	}
10675
10676	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10677	/* Paranoia. */
10678	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10679	#endif
10680
10681	/*
10682	* No longjmps to ring-3 from this point on!!!
10683	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10684	* This also disables flushing of the R0-logger instance (if any).
10685	*/
10686	VMMRZCallRing3Disable(pVCpu);
10687
10688	/*
10689	* Export the guest state bits.
10690	*
10691	* We cannot perform longjmps while loading the guest state because we do not preserve the
10692	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10693	* CPU migration.
10694	*
10695	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10696	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10697	*/
10698	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10699	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10700	{ /* likely */ }
10701	else
10702	{
10703	VMMRZCallRing3Enable(pVCpu);
10704	return rcStrict;
10705	}
10706
10707	/*
10708	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10709	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10710	* preemption disabled for a while. Since this is purely to aid the
10711	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10712	* disable interrupt on NT.
10713	*
10714	* We need to check for force-flags that could've possible been altered since we last
10715	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10716	* see @bugref{6398}).
10717	*
10718	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10719	* to ring-3 before executing guest code.
10720	*/
10721	pVmxTransient->fEFlags = ASMIntDisableFlags();
10722
10723	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10724	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10725	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10726	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10727	{
10728	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10729	{
10730	pVCpu->hm.s.Event.fPending = false;
10731
10732	/*
10733	* We've injected any pending events. This is really the point of no return (to ring-3).
10734	*
10735	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10736	* returns from this function, so don't enable them here.
10737	*/
10738	return VINF_SUCCESS;
10739	}
10740
10741	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10742	rcStrict = VINF_EM_RAW_INTERRUPT;
10743	}
10744	else
10745	{
10746	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10747	rcStrict = VINF_EM_RAW_TO_R3;
10748	}
10749
10750	ASMSetFlags(pVmxTransient->fEFlags);
10751	VMMRZCallRing3Enable(pVCpu);
10752
10753	return rcStrict;
10754	}
10755
10756
10757	/**
10758	* Final preparations before executing guest code using hardware-assisted VMX.
10759	*
10760	* We can no longer get preempted to a different host CPU and there are no returns
10761	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10762	* failures), this function is not intended to fail sans unrecoverable hardware
10763	* errors.
10764	*
10765	* @param pVCpu The cross context virtual CPU structure.
10766	* @param pVmxTransient The VMX-transient structure.
10767	*
10768	* @remarks Called with preemption disabled.
10769	* @remarks No-long-jump zone!!!
10770	*/
10771	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10772	{
10773	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10774	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10775	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10776	Assert(!pVCpu->hm.s.Event.fPending);
10777
10778	/*
10779	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10780	*/
10781	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10782	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10783
10784	PVM pVM = pVCpu->CTX_SUFF(pVM);
10785	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10786
10787	if (!CPUMIsGuestFPUStateActive(pVCpu))
10788	{
10789	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10790	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10791	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10792	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10793	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10794	}
10795
10796	/*
10797	* Re-save the host state bits as we may've been preempted (only happens when
10798	* thread-context hooks are used or when the VM start function changes).
10799	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10800	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10801	* see @bugref{8432}.
10802	*
10803	* This may also happen when switching to/from a nested-guest VMCS without leaving
10804	* ring-0.
10805	*/
10806	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10807	{
10808	int rc = hmR0VmxExportHostState(pVCpu);
10809	AssertRC(rc);
10810	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10811	}
10812	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10813
10814	/*
10815	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10816	*/
10817	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10818	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10819	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10820
10821	/*
10822	* Store status of the shared guest/host debug state at the time of VM-entry.
10823	*/
10824	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10825	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10826	{
10827	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10828	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10829	}
10830	else
10831	#endif
10832	{
10833	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10834	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10835	}
10836
10837	/*
10838	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10839	* more than one conditional check. The post-run side of our code shall determine
10840	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10841	*/
10842	if (pVmcsInfo->pbVirtApic)
10843	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10844
10845	/*
10846	* Update the host MSRs values in the VM-exit MSR-load area.
10847	*/
10848	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10849	{
10850	if (pVmcsInfo->cExitMsrLoad > 0)
10851	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10852	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10853	}
10854
10855	/*
10856	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10857	* VMX-preemption timer based on the next virtual sync clock deadline.
10858	*/
10859	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10860	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10861	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10862	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10863	{
10864	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10865	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10866	}
10867
10868	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10869	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10870	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10871	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10872
10873	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10874
10875	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10876	as we're about to start executing the guest . */
10877
10878	/*
10879	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10880	*
10881	* This is done this late as updating the TSC offsetting/preemption timer above
10882	* figures out if we can skip intercepting RDTSCP by calculating the number of
10883	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10884	*/
10885	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10886	{
10887	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10888	{
10889	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10890	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10891	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10892	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10893	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10894	AssertRC(rc);
10895	}
10896	else
10897	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10898	}
10899
10900	#ifdef VBOX_STRICT
10901	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10902	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10903	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10904	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10905	#endif
10906
10907	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10908	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10909	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10910	* see @bugref{9180#c54}. */
10911	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10912	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10913	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10914	#endif
10915	}
10916
10917
10918	/**
10919	* First C routine invoked after running guest code using hardware-assisted VMX.
10920	*
10921	* @param pVCpu The cross context virtual CPU structure.
10922	* @param pVmxTransient The VMX-transient structure.
10923	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10924	*
10925	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10926	*
10927	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10928	* unconditionally when it is safe to do so.
10929	*/
10930	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10931	{
10932	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10933
10934	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10935	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10936	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10937	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10938	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10939	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10940
10941	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10942	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10943	{
10944	uint64_t uGstTsc;
10945	if (!pVmxTransient->fIsNestedGuest)
10946	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10947	else
10948	{
10949	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10950	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10951	}
10952	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10953	}
10954
10955	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10956	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10957	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10958
10959	#if HC_ARCH_BITS == 64
10960	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10961	#endif
10962	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10963	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10964	and we need to leave it alone here. */
10965	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10966	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10967	#else
10968	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10969	#endif
10970	#ifdef VBOX_STRICT
10971	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10972	#endif
10973	Assert(!ASMIntAreEnabled());
10974	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10975	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10976
10977	/*
10978	* Save the basic VM-exit reason and check if the VM-entry failed.
10979	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10980	*/
10981	uint32_t uExitReason;
10982	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10983	AssertRC(rc);
10984	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10985	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10986
10987	/*
10988	* Check if VMLAUNCH/VMRESUME succeeded.
10989	* If this failed, we cause a guru meditation and cease further execution.
10990	*
10991	* However, if we are executing a nested-guest we might fail if we use the
10992	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10993	*/
10994	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10995	{
10996	/*
10997	* Update the VM-exit history array here even if the VM-entry failed due to:
10998	* - Invalid guest state.
10999	* - MSR loading.
11000	* - Machine-check event.
11001	*
11002	* In any of the above cases we will still have a "valid" VM-exit reason
11003	* despite @a fVMEntryFailed being false.
11004	*
11005	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11006	*
11007	* Note! We don't have CS or RIP at this point. Will probably address that later
11008	* by amending the history entry added here.
11009	*/
11010	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11011	UINT64_MAX, uHostTsc);
11012
11013	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11014	{
11015	VMMRZCallRing3Enable(pVCpu);
11016
11017	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11018	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11019
11020	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11021	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11022	AssertRC(rc);
11023	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11024	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11025	AssertRC(rc);
11026	#else
11027	/*
11028	* Import the guest-interruptibility state always as we need it while evaluating
11029	* injecting events on re-entry.
11030	*
11031	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11032	* checking for real-mode while exporting the state because all bits that cause
11033	* mode changes wrt CR0 are intercepted.
11034	*/
11035	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11036	AssertRC(rc);
11037	#endif
11038
11039	/*
11040	* Sync the TPR shadow with our APIC state.
11041	*/
11042	if ( !pVmxTransient->fIsNestedGuest
11043	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
11044	{
11045	Assert(pVmcsInfo->pbVirtApic);
11046	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11047	{
11048	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11049	AssertRC(rc);
11050	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11051	}
11052	}
11053
11054	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11055	return;
11056	}
11057	}
11058	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11059	else if (pVmxTransient->fIsNestedGuest)
11060	{
11061	# if 0
11062	/*
11063	* Copy the VM-instruction error field to the guest VMCS.
11064	*/
11065	/** @todo NSTVMX: Verify we're using the fast path. */
11066	uint32_t u32RoVmInstrError;
11067	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
11068	AssertRCReturn(rc, rc);
11069	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
11070	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
11071	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
11072	# else
11073	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11074	# endif
11075	}
11076	#endif
11077	else
11078	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11079
11080	VMMRZCallRing3Enable(pVCpu);
11081	}
11082
11083
11084	/**
11085	* Runs the guest code using hardware-assisted VMX the normal way.
11086	*
11087	* @returns VBox status code.
11088	* @param pVCpu The cross context virtual CPU structure.
11089	* @param pcLoops Pointer to the number of executed loops.
11090	*/
11091	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11092	{
11093	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11094	Assert(pcLoops);
11095	Assert(*pcLoops <= cMaxResumeLoops);
11096
11097	VMXTRANSIENT VmxTransient;
11098	RT_ZERO(VmxTransient);
11099	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11100
11101	/* Paranoia. */
11102	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11103	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11104
11105	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11106	for (;;)
11107	{
11108	Assert(!HMR0SuspendPending());
11109	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11110	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11111
11112	/*
11113	* Preparatory work for running nested-guest code, this may force us to
11114	* return to ring-3.
11115	*
11116	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11117	*/
11118	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11119	if (rcStrict != VINF_SUCCESS)
11120	break;
11121
11122	/* Interrupts are disabled at this point! */
11123	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11124	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11125	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11126	/* Interrupts are re-enabled at this point! */
11127
11128	/*
11129	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11130	*/
11131	if (RT_SUCCESS(rcRun))
11132	{ /* very likely */ }
11133	else
11134	{
11135	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11136	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11137	return rcRun;
11138	}
11139
11140	/*
11141	* Profile the VM-exit.
11142	*/
11143	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11144	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11145	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11146	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11147	HMVMX_START_EXIT_DISPATCH_PROF();
11148
11149	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11150
11151	/*
11152	* Handle the VM-exit.
11153	*/
11154	#ifdef HMVMX_USE_FUNCTION_TABLE
11155	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11156	#else
11157	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11158	#endif
11159	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11160	if (rcStrict == VINF_SUCCESS)
11161	{
11162	if (++(*pcLoops) <= cMaxResumeLoops)
11163	continue;
11164	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11165	rcStrict = VINF_EM_RAW_INTERRUPT;
11166	}
11167	break;
11168	}
11169
11170	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11171	return rcStrict;
11172	}
11173
11174	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11175	/**
11176	* Runs the nested-guest code using hardware-assisted VMX.
11177	*
11178	* @returns VBox status code.
11179	* @param pVCpu The cross context virtual CPU structure.
11180	* @param pcLoops Pointer to the number of executed loops.
11181	*
11182	* @sa hmR0VmxRunGuestCodeNormal.
11183	*/
11184	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11185	{
11186	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11187	Assert(pcLoops);
11188	Assert(*pcLoops <= cMaxResumeLoops);
11189
11190	VMXTRANSIENT VmxTransient;
11191	RT_ZERO(VmxTransient);
11192	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11193	VmxTransient.fIsNestedGuest = true;
11194
11195	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11196	for (;;)
11197	{
11198	Assert(!HMR0SuspendPending());
11199	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11200	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11201
11202	/*
11203	* Preparatory work for running guest code, this may force us to
11204	* return to ring-3.
11205	*
11206	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11207	*/
11208	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11209	if (rcStrict != VINF_SUCCESS)
11210	break;
11211
11212	/* Interrupts are disabled at this point! */
11213	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11214	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11215	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11216	/* Interrupts are re-enabled at this point! */
11217
11218	/*
11219	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11220	*/
11221	if (RT_SUCCESS(rcRun))
11222	{ /* very likely */ }
11223	else
11224	{
11225	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11226	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11227	return rcRun;
11228	}
11229
11230	/*
11231	* Profile the VM-exit.
11232	*/
11233	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11235	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11236	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11237	HMVMX_START_EXIT_DISPATCH_PROF();
11238
11239	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11240
11241	/*
11242	* Handle the VM-exit.
11243	*/
11244	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11245	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11246	if ( rcStrict == VINF_SUCCESS
11247	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11248	{
11249	if (++(*pcLoops) <= cMaxResumeLoops)
11250	continue;
11251	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11252	rcStrict = VINF_EM_RAW_INTERRUPT;
11253	}
11254	break;
11255	}
11256
11257	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11258	return rcStrict;
11259	}
11260	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11261
11262
11263	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11264	* probes.
11265	*
11266	* The following few functions and associated structure contains the bloat
11267	* necessary for providing detailed debug events and dtrace probes as well as
11268	* reliable host side single stepping. This works on the principle of
11269	* "subclassing" the normal execution loop and workers. We replace the loop
11270	* method completely and override selected helpers to add necessary adjustments
11271	* to their core operation.
11272	*
11273	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11274	* any performance for debug and analysis features.
11275	*
11276	* @{
11277	*/
11278
11279	/**
11280	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11281	* the debug run loop.
11282	*/
11283	typedef struct VMXRUNDBGSTATE
11284	{
11285	/** The RIP we started executing at. This is for detecting that we stepped. */
11286	uint64_t uRipStart;
11287	/** The CS we started executing with. */
11288	uint16_t uCsStart;
11289
11290	/** Whether we've actually modified the 1st execution control field. */
11291	bool fModifiedProcCtls : 1;
11292	/** Whether we've actually modified the 2nd execution control field. */
11293	bool fModifiedProcCtls2 : 1;
11294	/** Whether we've actually modified the exception bitmap. */
11295	bool fModifiedXcptBitmap : 1;
11296
11297	/** We desire the modified the CR0 mask to be cleared. */
11298	bool fClearCr0Mask : 1;
11299	/** We desire the modified the CR4 mask to be cleared. */
11300	bool fClearCr4Mask : 1;
11301	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11302	uint32_t fCpe1Extra;
11303	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11304	uint32_t fCpe1Unwanted;
11305	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11306	uint32_t fCpe2Extra;
11307	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11308	uint32_t bmXcptExtra;
11309	/** The sequence number of the Dtrace provider settings the state was
11310	* configured against. */
11311	uint32_t uDtraceSettingsSeqNo;
11312	/** VM-exits to check (one bit per VM-exit). */
11313	uint32_t bmExitsToCheck[3];
11314
11315	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11316	uint32_t fProcCtlsInitial;
11317	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11318	uint32_t fProcCtls2Initial;
11319	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11320	uint32_t bmXcptInitial;
11321	} VMXRUNDBGSTATE;
11322	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11323	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11324
11325
11326	/**
11327	* Initializes the VMXRUNDBGSTATE structure.
11328	*
11329	* @param pVCpu The cross context virtual CPU structure of the
11330	* calling EMT.
11331	* @param pVmxTransient The VMX-transient structure.
11332	* @param pDbgState The debug state to initialize.
11333	*/
11334	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11335	{
11336	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11337	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11338
11339	pDbgState->fModifiedProcCtls = false;
11340	pDbgState->fModifiedProcCtls2 = false;
11341	pDbgState->fModifiedXcptBitmap = false;
11342	pDbgState->fClearCr0Mask = false;
11343	pDbgState->fClearCr4Mask = false;
11344	pDbgState->fCpe1Extra = 0;
11345	pDbgState->fCpe1Unwanted = 0;
11346	pDbgState->fCpe2Extra = 0;
11347	pDbgState->bmXcptExtra = 0;
11348	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11349	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11350	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11351	}
11352
11353
11354	/**
11355	* Updates the VMSC fields with changes requested by @a pDbgState.
11356	*
11357	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11358	* immediately before executing guest code, i.e. when interrupts are disabled.
11359	* We don't check status codes here as we cannot easily assert or return in the
11360	* latter case.
11361	*
11362	* @param pVCpu The cross context virtual CPU structure.
11363	* @param pVmxTransient The VMX-transient structure.
11364	* @param pDbgState The debug state.
11365	*/
11366	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11367	{
11368	/*
11369	* Ensure desired flags in VMCS control fields are set.
11370	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11371	*
11372	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11373	* there should be no stale data in pCtx at this point.
11374	*/
11375	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11376	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11377	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11378	{
11379	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11380	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11381	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11382	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11383	pDbgState->fModifiedProcCtls = true;
11384	}
11385
11386	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11387	{
11388	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11389	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11390	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11391	pDbgState->fModifiedProcCtls2 = true;
11392	}
11393
11394	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11395	{
11396	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11397	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11398	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11399	pDbgState->fModifiedXcptBitmap = true;
11400	}
11401
11402	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11403	{
11404	pVmcsInfo->u64Cr0Mask = 0;
11405	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11406	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11407	}
11408
11409	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11410	{
11411	pVmcsInfo->u64Cr4Mask = 0;
11412	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11413	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11414	}
11415
11416	NOREF(pVCpu);
11417	}
11418
11419
11420	/**
11421	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11422	* re-entry next time around.
11423	*
11424	* @returns Strict VBox status code (i.e. informational status codes too).
11425	* @param pVCpu The cross context virtual CPU structure.
11426	* @param pVmxTransient The VMX-transient structure.
11427	* @param pDbgState The debug state.
11428	* @param rcStrict The return code from executing the guest using single
11429	* stepping.
11430	*/
11431	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11432	VBOXSTRICTRC rcStrict)
11433	{
11434	/*
11435	* Restore VM-exit control settings as we may not reenter this function the
11436	* next time around.
11437	*/
11438	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11439
11440	/* We reload the initial value, trigger what we can of recalculations the
11441	next time around. From the looks of things, that's all that's required atm. */
11442	if (pDbgState->fModifiedProcCtls)
11443	{
11444	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11445	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11446	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11447	AssertRCReturn(rc2, rc2);
11448	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11449	}
11450
11451	/* We're currently the only ones messing with this one, so just restore the
11452	cached value and reload the field. */
11453	if ( pDbgState->fModifiedProcCtls2
11454	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11455	{
11456	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11457	AssertRCReturn(rc2, rc2);
11458	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11459	}
11460
11461	/* If we've modified the exception bitmap, we restore it and trigger
11462	reloading and partial recalculation the next time around. */
11463	if (pDbgState->fModifiedXcptBitmap)
11464	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11465
11466	return rcStrict;
11467	}
11468
11469
11470	/**
11471	* Configures VM-exit controls for current DBGF and DTrace settings.
11472	*
11473	* This updates @a pDbgState and the VMCS execution control fields to reflect
11474	* the necessary VM-exits demanded by DBGF and DTrace.
11475	*
11476	* @param pVCpu The cross context virtual CPU structure.
11477	* @param pVmxTransient The VMX-transient structure. May update
11478	* fUpdatedTscOffsettingAndPreemptTimer.
11479	* @param pDbgState The debug state.
11480	*/
11481	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11482	{
11483	/*
11484	* Take down the dtrace serial number so we can spot changes.
11485	*/
11486	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11487	ASMCompilerBarrier();
11488
11489	/*
11490	* We'll rebuild most of the middle block of data members (holding the
11491	* current settings) as we go along here, so start by clearing it all.
11492	*/
11493	pDbgState->bmXcptExtra = 0;
11494	pDbgState->fCpe1Extra = 0;
11495	pDbgState->fCpe1Unwanted = 0;
11496	pDbgState->fCpe2Extra = 0;
11497	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11498	pDbgState->bmExitsToCheck[i] = 0;
11499
11500	/*
11501	* Software interrupts (INT XXh) - no idea how to trigger these...
11502	*/
11503	PVM pVM = pVCpu->CTX_SUFF(pVM);
11504	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11505	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11506	{
11507	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11508	}
11509
11510	/*
11511	* INT3 breakpoints - triggered by #BP exceptions.
11512	*/
11513	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11514	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11515
11516	/*
11517	* Exception bitmap and XCPT events+probes.
11518	*/
11519	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11520	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11521	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11522
11523	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11524	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11525	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11526	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11527	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11528	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11529	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11530	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11531	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11532	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11533	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11534	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11535	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11536	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11537	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11538	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11539	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11540	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11541
11542	if (pDbgState->bmXcptExtra)
11543	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11544
11545	/*
11546	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11547	*
11548	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11549	* So, when adding/changing/removing please don't forget to update it.
11550	*
11551	* Some of the macros are picking up local variables to save horizontal space,
11552	* (being able to see it in a table is the lesser evil here).
11553	*/
11554	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11555	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11556	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11557	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11558	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11559	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11560	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11561	} else do { } while (0)
11562	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11563	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11564	{ \
11565	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11566	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11567	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11568	} else do { } while (0)
11569	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11570	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11571	{ \
11572	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11573	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11574	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11575	} else do { } while (0)
11576	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11577	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11578	{ \
11579	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11580	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11581	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11582	} else do { } while (0)
11583
11584	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11585	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11586	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11587	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11588	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11589
11590	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11591	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11592	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11593	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11594	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11595	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11596	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11597	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11598	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11599	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11600	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11601	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11602	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11603	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11604	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11605	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11606	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11607	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11608	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11609	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11610	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11611	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11612	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11613	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11614	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11615	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11616	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11617	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11618	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11619	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11620	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11621	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11622	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11623	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11624	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11625	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11626
11627	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11628	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11629	{
11630	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11631	\| CPUMCTX_EXTRN_APIC_TPR);
11632	AssertRC(rc);
11633
11634	#if 0 /** @todo fix me */
11635	pDbgState->fClearCr0Mask = true;
11636	pDbgState->fClearCr4Mask = true;
11637	#endif
11638	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11639	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11640	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11641	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11642	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11643	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11644	require clearing here and in the loop if we start using it. */
11645	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11646	}
11647	else
11648	{
11649	if (pDbgState->fClearCr0Mask)
11650	{
11651	pDbgState->fClearCr0Mask = false;
11652	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11653	}
11654	if (pDbgState->fClearCr4Mask)
11655	{
11656	pDbgState->fClearCr4Mask = false;
11657	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11658	}
11659	}
11660	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11661	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11662
11663	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11664	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11665	{
11666	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11667	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11668	}
11669	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11670	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11671
11672	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11673	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11674	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11675	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11676	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11677	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11678	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11679	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11680	#if 0 /** @todo too slow, fix handler. */
11681	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11682	#endif
11683	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11684
11685	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11686	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11687	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11688	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11689	{
11690	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11691	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11692	}
11693	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11694	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11695	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11696	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11697
11698	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11699	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11700	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11701	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11702	{
11703	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11704	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11705	}
11706	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11707	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11708	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11709	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11710
11711	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11712	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11713	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11714	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11715	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11716	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11717	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11718	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11719	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11720	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11721	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11722	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11723	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11724	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11725	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11726	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11727	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11728	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11729	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11730	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11731	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11732	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11733
11734	#undef IS_EITHER_ENABLED
11735	#undef SET_ONLY_XBM_IF_EITHER_EN
11736	#undef SET_CPE1_XBM_IF_EITHER_EN
11737	#undef SET_CPEU_XBM_IF_EITHER_EN
11738	#undef SET_CPE2_XBM_IF_EITHER_EN
11739
11740	/*
11741	* Sanitize the control stuff.
11742	*/
11743	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11744	if (pDbgState->fCpe2Extra)
11745	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11746	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11747	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11748	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11749	{
11750	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11751	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11752	}
11753
11754	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11755	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11756	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11757	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11758	}
11759
11760
11761	/**
11762	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11763	* appropriate.
11764	*
11765	* The caller has checked the VM-exit against the
11766	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11767	* already, so we don't have to do that either.
11768	*
11769	* @returns Strict VBox status code (i.e. informational status codes too).
11770	* @param pVCpu The cross context virtual CPU structure.
11771	* @param pVmxTransient The VMX-transient structure.
11772	* @param uExitReason The VM-exit reason.
11773	*
11774	* @remarks The name of this function is displayed by dtrace, so keep it short
11775	* and to the point. No longer than 33 chars long, please.
11776	*/
11777	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11778	{
11779	/*
11780	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11781	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11782	*
11783	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11784	* does. Must add/change/remove both places. Same ordering, please.
11785	*
11786	* Added/removed events must also be reflected in the next section
11787	* where we dispatch dtrace events.
11788	*/
11789	bool fDtrace1 = false;
11790	bool fDtrace2 = false;
11791	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11792	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11793	uint32_t uEventArg = 0;
11794	#define SET_EXIT(a_EventSubName) \
11795	do { \
11796	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11797	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11798	} while (0)
11799	#define SET_BOTH(a_EventSubName) \
11800	do { \
11801	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11802	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11803	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11804	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11805	} while (0)
11806	switch (uExitReason)
11807	{
11808	case VMX_EXIT_MTF:
11809	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11810
11811	case VMX_EXIT_XCPT_OR_NMI:
11812	{
11813	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11814	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11815	{
11816	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11817	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11818	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11819	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11820	{
11821	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11822	{
11823	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11824	uEventArg = pVmxTransient->uExitIntErrorCode;
11825	}
11826	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11827	switch (enmEvent1)
11828	{
11829	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11830	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11831	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11832	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11833	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11834	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11835	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11836	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11837	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11838	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11839	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11840	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11841	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11842	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11843	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11844	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11845	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11846	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11847	default: break;
11848	}
11849	}
11850	else
11851	AssertFailed();
11852	break;
11853
11854	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11855	uEventArg = idxVector;
11856	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11857	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11858	break;
11859	}
11860	break;
11861	}
11862
11863	case VMX_EXIT_TRIPLE_FAULT:
11864	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11865	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11866	break;
11867	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11868	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11869	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11870	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11871	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11872
11873	/* Instruction specific VM-exits: */
11874	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11875	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11876	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11877	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11878	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11879	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11880	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11881	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11882	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11883	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11884	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11885	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11886	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11887	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11888	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11889	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11890	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11891	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11892	case VMX_EXIT_MOV_CRX:
11893	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11894	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11895	SET_BOTH(CRX_READ);
11896	else
11897	SET_BOTH(CRX_WRITE);
11898	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11899	break;
11900	case VMX_EXIT_MOV_DRX:
11901	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11902	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11903	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11904	SET_BOTH(DRX_READ);
11905	else
11906	SET_BOTH(DRX_WRITE);
11907	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11908	break;
11909	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11910	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11911	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11912	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11913	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11914	case VMX_EXIT_GDTR_IDTR_ACCESS:
11915	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11916	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11917	{
11918	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11919	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11920	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11921	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11922	}
11923	break;
11924
11925	case VMX_EXIT_LDTR_TR_ACCESS:
11926	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11927	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11928	{
11929	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11930	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11931	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11932	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11933	}
11934	break;
11935
11936	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11937	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11938	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11939	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11940	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11941	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11942	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11943	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11944	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11945	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11946	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11947
11948	/* Events that aren't relevant at this point. */
11949	case VMX_EXIT_EXT_INT:
11950	case VMX_EXIT_INT_WINDOW:
11951	case VMX_EXIT_NMI_WINDOW:
11952	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11953	case VMX_EXIT_PREEMPT_TIMER:
11954	case VMX_EXIT_IO_INSTR:
11955	break;
11956
11957	/* Errors and unexpected events. */
11958	case VMX_EXIT_INIT_SIGNAL:
11959	case VMX_EXIT_SIPI:
11960	case VMX_EXIT_IO_SMI:
11961	case VMX_EXIT_SMI:
11962	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11963	case VMX_EXIT_ERR_MSR_LOAD:
11964	case VMX_EXIT_ERR_MACHINE_CHECK:
11965	case VMX_EXIT_PML_FULL:
11966	case VMX_EXIT_VIRTUALIZED_EOI:
11967	break;
11968
11969	default:
11970	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11971	break;
11972	}
11973	#undef SET_BOTH
11974	#undef SET_EXIT
11975
11976	/*
11977	* Dtrace tracepoints go first. We do them here at once so we don't
11978	* have to copy the guest state saving and stuff a few dozen times.
11979	* Down side is that we've got to repeat the switch, though this time
11980	* we use enmEvent since the probes are a subset of what DBGF does.
11981	*/
11982	if (fDtrace1 \|\| fDtrace2)
11983	{
11984	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11985	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11986	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11987	switch (enmEvent1)
11988	{
11989	/** @todo consider which extra parameters would be helpful for each probe. */
11990	case DBGFEVENT_END: break;
11991	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11992	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11993	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11994	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11995	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11996	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11997	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11998	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11999	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12000	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12001	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12002	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12003	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12004	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12005	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12006	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12007	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12008	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12009	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12010	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12011	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12012	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12013	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12014	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12015	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12016	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12017	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12018	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12019	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12020	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12021	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12022	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12023	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12024	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12025	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12026	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12027	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12028	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12029	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12030	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12031	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12032	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12033	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12034	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12035	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12036	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12037	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12038	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12039	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12040	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12041	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12042	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12043	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12044	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12045	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12046	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12047	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12048	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12049	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12050	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12051	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12052	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12053	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12054	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12055	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12056	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12057	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12058	}
12059	switch (enmEvent2)
12060	{
12061	/** @todo consider which extra parameters would be helpful for each probe. */
12062	case DBGFEVENT_END: break;
12063	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12064	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12065	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12066	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12067	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12068	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12069	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12070	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12071	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12072	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12073	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12074	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12075	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12076	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12077	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12078	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12079	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12080	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12081	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12082	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12083	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12084	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12085	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12086	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12087	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12088	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12089	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12090	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12091	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12092	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12093	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12094	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12095	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12096	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12097	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12098	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12099	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12100	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12101	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12102	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12103	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12104	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12105	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12106	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12107	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12108	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12109	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12110	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12111	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12112	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12113	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12114	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12115	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12116	}
12117	}
12118
12119	/*
12120	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12121	* the DBGF call will do a full check).
12122	*
12123	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12124	* Note! If we have to events, we prioritize the first, i.e. the instruction
12125	* one, in order to avoid event nesting.
12126	*/
12127	PVM pVM = pVCpu->CTX_SUFF(pVM);
12128	if ( enmEvent1 != DBGFEVENT_END
12129	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12130	{
12131	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12132	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12133	if (rcStrict != VINF_SUCCESS)
12134	return rcStrict;
12135	}
12136	else if ( enmEvent2 != DBGFEVENT_END
12137	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12138	{
12139	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12140	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12141	if (rcStrict != VINF_SUCCESS)
12142	return rcStrict;
12143	}
12144
12145	return VINF_SUCCESS;
12146	}
12147
12148
12149	/**
12150	* Single-stepping VM-exit filtering.
12151	*
12152	* This is preprocessing the VM-exits and deciding whether we've gotten far
12153	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12154	* handling is performed.
12155	*
12156	* @returns Strict VBox status code (i.e. informational status codes too).
12157	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12158	* @param pVmxTransient The VMX-transient structure.
12159	* @param pDbgState The debug state.
12160	*/
12161	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12162	{
12163	/*
12164	* Expensive (saves context) generic dtrace VM-exit probe.
12165	*/
12166	uint32_t const uExitReason = pVmxTransient->uExitReason;
12167	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12168	{ /* more likely */ }
12169	else
12170	{
12171	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12172	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12173	AssertRC(rc);
12174	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12175	}
12176
12177	/*
12178	* Check for host NMI, just to get that out of the way.
12179	*/
12180	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12181	{ /* normally likely */ }
12182	else
12183	{
12184	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12185	AssertRCReturn(rc2, rc2);
12186	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12187	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12188	return hmR0VmxExitHostNmi(pVCpu);
12189	}
12190
12191	/*
12192	* Check for single stepping event if we're stepping.
12193	*/
12194	if (pVCpu->hm.s.fSingleInstruction)
12195	{
12196	switch (uExitReason)
12197	{
12198	case VMX_EXIT_MTF:
12199	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12200
12201	/* Various events: */
12202	case VMX_EXIT_XCPT_OR_NMI:
12203	case VMX_EXIT_EXT_INT:
12204	case VMX_EXIT_TRIPLE_FAULT:
12205	case VMX_EXIT_INT_WINDOW:
12206	case VMX_EXIT_NMI_WINDOW:
12207	case VMX_EXIT_TASK_SWITCH:
12208	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12209	case VMX_EXIT_APIC_ACCESS:
12210	case VMX_EXIT_EPT_VIOLATION:
12211	case VMX_EXIT_EPT_MISCONFIG:
12212	case VMX_EXIT_PREEMPT_TIMER:
12213
12214	/* Instruction specific VM-exits: */
12215	case VMX_EXIT_CPUID:
12216	case VMX_EXIT_GETSEC:
12217	case VMX_EXIT_HLT:
12218	case VMX_EXIT_INVD:
12219	case VMX_EXIT_INVLPG:
12220	case VMX_EXIT_RDPMC:
12221	case VMX_EXIT_RDTSC:
12222	case VMX_EXIT_RSM:
12223	case VMX_EXIT_VMCALL:
12224	case VMX_EXIT_VMCLEAR:
12225	case VMX_EXIT_VMLAUNCH:
12226	case VMX_EXIT_VMPTRLD:
12227	case VMX_EXIT_VMPTRST:
12228	case VMX_EXIT_VMREAD:
12229	case VMX_EXIT_VMRESUME:
12230	case VMX_EXIT_VMWRITE:
12231	case VMX_EXIT_VMXOFF:
12232	case VMX_EXIT_VMXON:
12233	case VMX_EXIT_MOV_CRX:
12234	case VMX_EXIT_MOV_DRX:
12235	case VMX_EXIT_IO_INSTR:
12236	case VMX_EXIT_RDMSR:
12237	case VMX_EXIT_WRMSR:
12238	case VMX_EXIT_MWAIT:
12239	case VMX_EXIT_MONITOR:
12240	case VMX_EXIT_PAUSE:
12241	case VMX_EXIT_GDTR_IDTR_ACCESS:
12242	case VMX_EXIT_LDTR_TR_ACCESS:
12243	case VMX_EXIT_INVEPT:
12244	case VMX_EXIT_RDTSCP:
12245	case VMX_EXIT_INVVPID:
12246	case VMX_EXIT_WBINVD:
12247	case VMX_EXIT_XSETBV:
12248	case VMX_EXIT_RDRAND:
12249	case VMX_EXIT_INVPCID:
12250	case VMX_EXIT_VMFUNC:
12251	case VMX_EXIT_RDSEED:
12252	case VMX_EXIT_XSAVES:
12253	case VMX_EXIT_XRSTORS:
12254	{
12255	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12256	AssertRCReturn(rc, rc);
12257	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12258	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12259	return VINF_EM_DBG_STEPPED;
12260	break;
12261	}
12262
12263	/* Errors and unexpected events: */
12264	case VMX_EXIT_INIT_SIGNAL:
12265	case VMX_EXIT_SIPI:
12266	case VMX_EXIT_IO_SMI:
12267	case VMX_EXIT_SMI:
12268	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12269	case VMX_EXIT_ERR_MSR_LOAD:
12270	case VMX_EXIT_ERR_MACHINE_CHECK:
12271	case VMX_EXIT_PML_FULL:
12272	case VMX_EXIT_VIRTUALIZED_EOI:
12273	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12274	break;
12275
12276	default:
12277	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12278	break;
12279	}
12280	}
12281
12282	/*
12283	* Check for debugger event breakpoints and dtrace probes.
12284	*/
12285	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12286	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12287	{
12288	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12289	if (rcStrict != VINF_SUCCESS)
12290	return rcStrict;
12291	}
12292
12293	/*
12294	* Normal processing.
12295	*/
12296	#ifdef HMVMX_USE_FUNCTION_TABLE
12297	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12298	#else
12299	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12300	#endif
12301	}
12302
12303
12304	/**
12305	* Single steps guest code using hardware-assisted VMX.
12306	*
12307	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12308	* but single-stepping through the hypervisor debugger.
12309	*
12310	* @returns Strict VBox status code (i.e. informational status codes too).
12311	* @param pVCpu The cross context virtual CPU structure.
12312	* @param pcLoops Pointer to the number of executed loops.
12313	*
12314	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12315	*/
12316	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12317	{
12318	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12319	Assert(pcLoops);
12320	Assert(*pcLoops <= cMaxResumeLoops);
12321
12322	VMXTRANSIENT VmxTransient;
12323	RT_ZERO(VmxTransient);
12324	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12325
12326	/* Set HMCPU indicators. */
12327	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12328	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12329	pVCpu->hm.s.fDebugWantRdTscExit = false;
12330	pVCpu->hm.s.fUsingDebugLoop = true;
12331
12332	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12333	VMXRUNDBGSTATE DbgState;
12334	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12335	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12336
12337	/*
12338	* The loop.
12339	*/
12340	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12341	for (;;)
12342	{
12343	Assert(!HMR0SuspendPending());
12344	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12345	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12346	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12347
12348	/* Set up VM-execution controls the next two can respond to. */
12349	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12350
12351	/*
12352	* Preparatory work for running guest code, this may force us to
12353	* return to ring-3.
12354	*
12355	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12356	*/
12357	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12358	if (rcStrict != VINF_SUCCESS)
12359	break;
12360
12361	/* Interrupts are disabled at this point! */
12362	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12363
12364	/* Override any obnoxious code in the above two calls. */
12365	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12366
12367	/*
12368	* Finally execute the guest.
12369	*/
12370	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12371
12372	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12373	/* Interrupts are re-enabled at this point! */
12374
12375	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12376	if (RT_SUCCESS(rcRun))
12377	{ /* very likely */ }
12378	else
12379	{
12380	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12381	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12382	return rcRun;
12383	}
12384
12385	/* Profile the VM-exit. */
12386	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12388	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12389	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12390	HMVMX_START_EXIT_DISPATCH_PROF();
12391
12392	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12393
12394	/*
12395	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12396	*/
12397	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12398	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12399	if (rcStrict != VINF_SUCCESS)
12400	break;
12401	if (++(*pcLoops) > cMaxResumeLoops)
12402	{
12403	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12404	rcStrict = VINF_EM_RAW_INTERRUPT;
12405	break;
12406	}
12407
12408	/*
12409	* Stepping: Did the RIP change, if so, consider it a single step.
12410	* Otherwise, make sure one of the TFs gets set.
12411	*/
12412	if (fStepping)
12413	{
12414	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12415	AssertRC(rc);
12416	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12417	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12418	{
12419	rcStrict = VINF_EM_DBG_STEPPED;
12420	break;
12421	}
12422	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12423	}
12424
12425	/*
12426	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12427	*/
12428	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12429	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12430	}
12431
12432	/*
12433	* Clear the X86_EFL_TF if necessary.
12434	*/
12435	if (pVCpu->hm.s.fClearTrapFlag)
12436	{
12437	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12438	AssertRC(rc);
12439	pVCpu->hm.s.fClearTrapFlag = false;
12440	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12441	}
12442	/** @todo there seems to be issues with the resume flag when the monitor trap
12443	* flag is pending without being used. Seen early in bios init when
12444	* accessing APIC page in protected mode. */
12445
12446	/*
12447	* Restore VM-exit control settings as we may not re-enter this function the
12448	* next time around.
12449	*/
12450	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12451
12452	/* Restore HMCPU indicators. */
12453	pVCpu->hm.s.fUsingDebugLoop = false;
12454	pVCpu->hm.s.fDebugWantRdTscExit = false;
12455	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12456
12457	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12458	return rcStrict;
12459	}
12460
12461
12462	/** @} */
12463
12464
12465	/**
12466	* Checks if any expensive dtrace probes are enabled and we should go to the
12467	* debug loop.
12468	*
12469	* @returns true if we should use debug loop, false if not.
12470	*/
12471	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12472	{
12473	/* It's probably faster to OR the raw 32-bit counter variables together.
12474	Since the variables are in an array and the probes are next to one
12475	another (more or less), we have good locality. So, better read
12476	eight-nine cache lines ever time and only have one conditional, than
12477	128+ conditionals, right? */
12478	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12479	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12480	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12481	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12482	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12483	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12484	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12485	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12486	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12487	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12488	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12489	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12490	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12491	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12492	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12493	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12494	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12495	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12496	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12497	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12498	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12499	) != 0
12500	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12501	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12502	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12503	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12504	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12505	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12506	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12507	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12508	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12509	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12510	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12511	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12512	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12513	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12514	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12515	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12516	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12517	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12518	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12519	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12520	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12521	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12522	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12523	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12524	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12525	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12526	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12527	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12528	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12529	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12530	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12531	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12532	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12533	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12534	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12535	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12536	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12537	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12538	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12539	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12540	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12541	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12542	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12543	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12544	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12545	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12546	) != 0
12547	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12548	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12549	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12550	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12551	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12552	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12553	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12554	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12555	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12556	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12557	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12558	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12559	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12560	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12561	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12562	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12563	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12564	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12565	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12566	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12567	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12568	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12569	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12570	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12571	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12572	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12573	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12574	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12575	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12576	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12577	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12578	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12579	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12580	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12581	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12582	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12583	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12584	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12585	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12586	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12587	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12588	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12589	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12590	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12591	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12592	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12593	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12594	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12595	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12596	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12597	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12598	) != 0;
12599	}
12600
12601
12602	/**
12603	* Runs the guest using hardware-assisted VMX.
12604	*
12605	* @returns Strict VBox status code (i.e. informational status codes too).
12606	* @param pVCpu The cross context virtual CPU structure.
12607	*/
12608	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12609	{
12610	AssertPtr(pVCpu);
12611	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12612	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12613	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12614	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12615
12616	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12617
12618	VBOXSTRICTRC rcStrict;
12619	uint32_t cLoops = 0;
12620	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12621	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12622	#else
12623	bool const fInNestedGuestMode = false;
12624	#endif
12625	if (!fInNestedGuestMode)
12626	{
12627	if ( !pVCpu->hm.s.fUseDebugLoop
12628	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12629	&& !DBGFIsStepping(pVCpu)
12630	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12631	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12632	else
12633	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12634	}
12635	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12636	else
12637	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12638
12639	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12640	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12641	#endif
12642
12643	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12644	switch (rcLoop)
12645	{
12646	case VINF_VMX_VMEXIT: rcStrict = VINF_SUCCESS; break;
12647	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12648	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12649	}
12650
12651	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12652	if (RT_FAILURE(rc2))
12653	{
12654	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12655	rcStrict = rc2;
12656	}
12657	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12658	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12659	return rcStrict;
12660	}
12661
12662
12663	#ifndef HMVMX_USE_FUNCTION_TABLE
12664	/**
12665	* Handles a guest VM-exit from hardware-assisted VMX execution.
12666	*
12667	* @returns Strict VBox status code (i.e. informational status codes too).
12668	* @param pVCpu The cross context virtual CPU structure.
12669	* @param pVmxTransient The VMX-transient structure.
12670	*/
12671	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12672	{
12673	#ifdef DEBUG_ramshankar
12674	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12675	do { \
12676	if (a_fSave != 0) \
12677	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12678	VBOXSTRICTRC rcStrict = a_CallExpr; \
12679	if (a_fSave != 0) \
12680	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12681	return rcStrict; \
12682	} while (0)
12683	#else
12684	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12685	#endif
12686	uint32_t const uExitReason = pVmxTransient->uExitReason;
12687	switch (uExitReason)
12688	{
12689	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12690	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12691	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12692	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12693	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12694	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12695	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12696	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12697	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12698	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12699	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12700	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12701	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12702	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12703	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12704	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12705	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12706	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12707	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12708	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12709	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12710	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12711	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12712	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12713	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12714	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12715	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12716	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12717	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12718	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12719	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12720	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12721	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12722	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12723	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12724	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12725	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12726	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12727	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12728	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12729	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12730	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12731	#else
12732	case VMX_EXIT_VMCLEAR:
12733	case VMX_EXIT_VMLAUNCH:
12734	case VMX_EXIT_VMPTRLD:
12735	case VMX_EXIT_VMPTRST:
12736	case VMX_EXIT_VMREAD:
12737	case VMX_EXIT_VMRESUME:
12738	case VMX_EXIT_VMWRITE:
12739	case VMX_EXIT_VMXOFF:
12740	case VMX_EXIT_VMXON:
12741	case VMX_EXIT_INVVPID:
12742	case VMX_EXIT_INVEPT:
12743	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12744	#endif
12745
12746	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12747	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12748	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12749
12750	case VMX_EXIT_INIT_SIGNAL:
12751	case VMX_EXIT_SIPI:
12752	case VMX_EXIT_IO_SMI:
12753	case VMX_EXIT_SMI:
12754	case VMX_EXIT_ERR_MSR_LOAD:
12755	case VMX_EXIT_ERR_MACHINE_CHECK:
12756	case VMX_EXIT_PML_FULL:
12757	case VMX_EXIT_VIRTUALIZED_EOI:
12758	case VMX_EXIT_GDTR_IDTR_ACCESS:
12759	case VMX_EXIT_LDTR_TR_ACCESS:
12760	case VMX_EXIT_APIC_WRITE:
12761	case VMX_EXIT_RDRAND:
12762	case VMX_EXIT_RSM:
12763	case VMX_EXIT_VMFUNC:
12764	case VMX_EXIT_ENCLS:
12765	case VMX_EXIT_RDSEED:
12766	case VMX_EXIT_XSAVES:
12767	case VMX_EXIT_XRSTORS:
12768	case VMX_EXIT_UMWAIT:
12769	case VMX_EXIT_TPAUSE:
12770	default:
12771	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12772	}
12773	#undef VMEXIT_CALL_RET
12774	}
12775	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12776
12777
12778	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12779	/**
12780	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12781	*
12782	* @returns Strict VBox status code (i.e. informational status codes too).
12783	* @param pVCpu The cross context virtual CPU structure.
12784	* @param pVmxTransient The VMX-transient structure.
12785	*/
12786	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12787	{
12788	uint32_t const uExitReason = pVmxTransient->uExitReason;
12789	switch (uExitReason)
12790	{
12791	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12792	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12793	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12794	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12795	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12796
12797	/*
12798	* We shouldn't direct host physical interrupts to the nested-guest.
12799	*/
12800	case VMX_EXIT_EXT_INT:
12801	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12802
12803	/*
12804	* Instructions that cause VM-exits unconditionally or the condition is
12805	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12806	* happens, it's guaranteed to be a nested-guest VM-exit).
12807	*
12808	* - Provides VM-exit instruction length ONLY.
12809	*/
12810	case VMX_EXIT_CPUID: /* Unconditional. */
12811	case VMX_EXIT_VMCALL:
12812	case VMX_EXIT_GETSEC:
12813	case VMX_EXIT_INVD:
12814	case VMX_EXIT_XSETBV:
12815	case VMX_EXIT_VMLAUNCH:
12816	case VMX_EXIT_VMRESUME:
12817	case VMX_EXIT_VMXOFF:
12818	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
12819	case VMX_EXIT_VMFUNC:
12820	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12821
12822	/*
12823	* Instructions that cause VM-exits unconditionally or the condition is
12824	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12825	* happens, it's guaranteed to be a nested-guest VM-exit).
12826	*
12827	* - Provides VM-exit instruction length.
12828	* - Provides VM-exit information.
12829	* - Optionally provides Exit qualification.
12830	*
12831	* Since Exit qualification is 0 for all VM-exits where it is not
12832	* applicable, reading and passing it to the guest should produce
12833	* defined behavior.
12834	*
12835	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12836	*/
12837	case VMX_EXIT_INVEPT: /* Unconditional. */
12838	case VMX_EXIT_INVVPID:
12839	case VMX_EXIT_VMCLEAR:
12840	case VMX_EXIT_VMPTRLD:
12841	case VMX_EXIT_VMPTRST:
12842	case VMX_EXIT_VMXON:
12843	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
12844	case VMX_EXIT_LDTR_TR_ACCESS:
12845	case VMX_EXIT_RDRAND:
12846	case VMX_EXIT_RDSEED:
12847	case VMX_EXIT_XSAVES:
12848	case VMX_EXIT_XRSTORS:
12849	case VMX_EXIT_UMWAIT:
12850	case VMX_EXIT_TPAUSE:
12851	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12852
12853	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12854	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12855	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12856	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12857	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12858	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12859	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12860	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12861	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12862	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12863	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12864	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12865	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12866	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12867	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12868	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12869	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12870	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12871	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12872
12873	case VMX_EXIT_PREEMPT_TIMER:
12874	{
12875	/** @todo NSTVMX: Preempt timer. */
12876	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12877	}
12878
12879	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12880	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12881
12882	case VMX_EXIT_VMREAD:
12883	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12884
12885	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12886	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12887
12888	case VMX_EXIT_INIT_SIGNAL:
12889	case VMX_EXIT_SIPI:
12890	case VMX_EXIT_IO_SMI:
12891	case VMX_EXIT_SMI:
12892	case VMX_EXIT_ERR_MSR_LOAD:
12893	case VMX_EXIT_ERR_MACHINE_CHECK:
12894	case VMX_EXIT_PML_FULL:
12895	case VMX_EXIT_RSM:
12896	default:
12897	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12898	}
12899	}
12900	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12901
12902
12903	#ifdef VBOX_STRICT
12904	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12905	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12906	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12907
12908	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12909	do { \
12910	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12911	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12912	} while (0)
12913
12914	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12915	do { \
12916	AssertPtr((a_pVCpu)); \
12917	AssertPtr((a_pVmxTransient)); \
12918	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12919	Assert((a_pVmxTransient)->pVmcsInfo); \
12920	Assert(ASMIntAreEnabled()); \
12921	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12922	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12923	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12924	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12925	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12926	HMVMX_ASSERT_PREEMPT_CPUID(); \
12927	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12928	} while (0)
12929
12930	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12931	do { \
12932	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12933	Assert((a_pVmxTransient)->fIsNestedGuest); \
12934	} while (0)
12935
12936	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12937	do { \
12938	Log4Func(("\n")); \
12939	} while (0)
12940	#else
12941	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12942	do { \
12943	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12944	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12945	} while (0)
12946
12947	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12948	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12949
12950	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12951	#endif
12952
12953
12954	/**
12955	* Advances the guest RIP by the specified number of bytes.
12956	*
12957	* @param pVCpu The cross context virtual CPU structure.
12958	* @param cbInstr Number of bytes to advance the RIP by.
12959	*
12960	* @remarks No-long-jump zone!!!
12961	*/
12962	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12963	{
12964	/* Advance the RIP. */
12965	pVCpu->cpum.GstCtx.rip += cbInstr;
12966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12967
12968	/* Update interrupt inhibition. */
12969	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12970	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12971	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12972	}
12973
12974
12975	/**
12976	* Advances the guest RIP after reading it from the VMCS.
12977	*
12978	* @returns VBox status code, no informational status codes.
12979	* @param pVCpu The cross context virtual CPU structure.
12980	* @param pVmxTransient The VMX-transient structure.
12981	*
12982	* @remarks No-long-jump zone!!!
12983	*/
12984	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12985	{
12986	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12987	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12988	AssertRCReturn(rc, rc);
12989
12990	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12991	return VINF_SUCCESS;
12992	}
12993
12994
12995	/**
12996	* Handle a condition that occurred while delivering an event through the guest
12997	* IDT.
12998	*
12999	* @returns Strict VBox status code (i.e. informational status codes too).
13000	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13001	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13002	* to continue execution of the guest which will delivery the \#DF.
13003	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13004	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13005	*
13006	* @param pVCpu The cross context virtual CPU structure.
13007	* @param pVmxTransient The VMX-transient structure.
13008	*
13009	* @remarks No-long-jump zone!!!
13010	*/
13011	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13012	{
13013	/* Read the IDT vectoring info. and VM-exit interruption info. */
13014	{
13015	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13016	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13017	AssertRCReturn(rc, rc);
13018	}
13019
13020	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13021	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13022	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13023	{
13024	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13025	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13026
13027	/*
13028	* If the event was a software interrupt (generated with INT n) or a software exception
13029	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13030	* can handle the VM-exit and continue guest execution which will re-execute the
13031	* instruction rather than re-injecting the exception, as that can cause premature
13032	* trips to ring-3 before injection and involve TRPM which currently has no way of
13033	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13034	* the problem).
13035	*/
13036	IEMXCPTRAISE enmRaise;
13037	IEMXCPTRAISEINFO fRaiseInfo;
13038	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13039	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13040	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13041	{
13042	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13043	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13044	}
13045	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13046	{
13047	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13048	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13049	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13050
13051	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13052	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13053	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13054
13055	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13056
13057	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13058	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13059	{
13060	pVmxTransient->fVectoringPF = true;
13061	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13062	}
13063	}
13064	else
13065	{
13066	/*
13067	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13068	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13069	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13070	*/
13071	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13072	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13073	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13074	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13075	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13076	}
13077
13078	/*
13079	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13080	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13081	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13082	* subsequent VM-entry would fail.
13083	*
13084	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
13085	*/
13086	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13087	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13088	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13089	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13090	&& CPUMIsGuestNmiBlocking(pVCpu))
13091	{
13092	CPUMSetGuestNmiBlocking(pVCpu, false);
13093	}
13094
13095	switch (enmRaise)
13096	{
13097	case IEMXCPTRAISE_CURRENT_XCPT:
13098	{
13099	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13100	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13101	Assert(rcStrict == VINF_SUCCESS);
13102	break;
13103	}
13104
13105	case IEMXCPTRAISE_PREV_EVENT:
13106	{
13107	uint32_t u32ErrCode;
13108	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13109	{
13110	int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13111	AssertRCReturn(rc, rc);
13112	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13113	}
13114	else
13115	u32ErrCode = 0;
13116
13117	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13118	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13119	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13120	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13121
13122	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13123	pVCpu->hm.s.Event.u32ErrCode));
13124	Assert(rcStrict == VINF_SUCCESS);
13125	break;
13126	}
13127
13128	case IEMXCPTRAISE_REEXEC_INSTR:
13129	Assert(rcStrict == VINF_SUCCESS);
13130	break;
13131
13132	case IEMXCPTRAISE_DOUBLE_FAULT:
13133	{
13134	/*
13135	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13136	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13137	*/
13138	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13139	{
13140	pVmxTransient->fVectoringDoublePF = true;
13141	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13142	pVCpu->cpum.GstCtx.cr2));
13143	rcStrict = VINF_SUCCESS;
13144	}
13145	else
13146	{
13147	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13148	hmR0VmxSetPendingXcptDF(pVCpu);
13149	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13150	uIdtVector, uExitVector));
13151	rcStrict = VINF_HM_DOUBLE_FAULT;
13152	}
13153	break;
13154	}
13155
13156	case IEMXCPTRAISE_TRIPLE_FAULT:
13157	{
13158	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13159	rcStrict = VINF_EM_RESET;
13160	break;
13161	}
13162
13163	case IEMXCPTRAISE_CPU_HANG:
13164	{
13165	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13166	rcStrict = VERR_EM_GUEST_CPU_HANG;
13167	break;
13168	}
13169
13170	default:
13171	{
13172	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13173	rcStrict = VERR_VMX_IPE_2;
13174	break;
13175	}
13176	}
13177	}
13178	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13179	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13180	&& uExitVector != X86_XCPT_DF
13181	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13182	{
13183	/*
13184	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13185	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13186	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13187	*/
13188	CPUMSetGuestNmiBlocking(pVCpu, true);
13189	Log4Func(("Set NMI blocking. fValid=%RTbool uExitReason=%u\n", VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo),
13190	pVmxTransient->uExitReason));
13191	}
13192
13193	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13194	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13195	return rcStrict;
13196	}
13197
13198
13199	/** @name VM-exit handlers.
13200	* @{
13201	*/
13202	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13203	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13204	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13205
13206	/**
13207	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13208	*/
13209	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13210	{
13211	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13212	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13213	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13214	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13215	return VINF_SUCCESS;
13216	return VINF_EM_RAW_INTERRUPT;
13217	}
13218
13219
13220	/**
13221	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
13222	* VM-exit.
13223	*/
13224	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13225	{
13226	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13227	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13228
13229	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13230	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13231	AssertRCReturn(rc, rc);
13232
13233	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13234	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13235	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13236	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13237
13238	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13239	{
13240	/*
13241	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13242	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13243	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13244	*
13245	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13246	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13247	*/
13248	return hmR0VmxExitHostNmi(pVCpu);
13249	}
13250
13251	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13252	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13253	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
13254	{ /* likely */ }
13255	else
13256	{
13257	if (rcStrict == VINF_HM_DOUBLE_FAULT)
13258	rcStrict = VINF_SUCCESS;
13259	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13260	return rcStrict;
13261	}
13262
13263	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13264	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13265	switch (uIntType)
13266	{
13267	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13268	Assert(uVector == X86_XCPT_DB);
13269	RT_FALL_THRU();
13270	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13271	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13272	RT_FALL_THRU();
13273	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13274	{
13275	/*
13276	* If there's any exception caused as a result of event injection, the resulting
13277	* secondary/final execption will be pending, we shall continue guest execution
13278	* after injecting the event. The page-fault case is complicated and we manually
13279	* handle any currently pending event in hmR0VmxExitXcptPF.
13280	*/
13281	if (!pVCpu->hm.s.Event.fPending)
13282	{ /* likely */ }
13283	else if (uVector != X86_XCPT_PF)
13284	{
13285	rcStrict = VINF_SUCCESS;
13286	break;
13287	}
13288
13289	switch (uVector)
13290	{
13291	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13292	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13293	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13294	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13295	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13296	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13297
13298	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13299	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13300	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13301	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13302	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13303	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13304	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13305	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13306	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13307	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13308	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13309	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13310	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13311	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13312	default:
13313	{
13314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13315	if (pVmcsInfo->RealMode.fRealOnV86Active)
13316	{
13317	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13318	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13319	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13320
13321	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13322	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13323	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13324	AssertRCReturn(rc, rc);
13325	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13326	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13327	0 /* GCPtrFaultAddress */);
13328	rcStrict = VINF_SUCCESS;
13329	}
13330	else
13331	{
13332	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13333	pVCpu->hm.s.u32HMError = uVector;
13334	rcStrict = VERR_VMX_UNEXPECTED_EXCEPTION;
13335	}
13336	break;
13337	}
13338	}
13339	break;
13340	}
13341
13342	default:
13343	{
13344	pVCpu->hm.s.u32HMError = uExitIntInfo;
13345	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13346	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13347	break;
13348	}
13349	}
13350	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13351	return rcStrict;
13352	}
13353
13354
13355	/**
13356	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13357	*/
13358	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13359	{
13360	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13361
13362	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13363	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13364	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13365	AssertRCReturn(rc, rc);
13366
13367	/* Evaluate and deliver pending events and resume guest execution. */
13368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13369	return VINF_SUCCESS;
13370	}
13371
13372
13373	/**
13374	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13375	*/
13376	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13377	{
13378	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13379
13380	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13381	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13382	{
13383	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13384	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13385	}
13386
13387	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
13388
13389	/*
13390	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13391	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13392	*/
13393	uint32_t fIntrState;
13394	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13395	AssertRCReturn(rc, rc);
13396	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13397	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13398	{
13399	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13400	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13401
13402	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13403	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13404	AssertRCReturn(rc, rc);
13405	}
13406
13407	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13408	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13409	AssertRCReturn(rc, rc);
13410
13411	/* Evaluate and deliver pending events and resume guest execution. */
13412	return VINF_SUCCESS;
13413	}
13414
13415
13416	/**
13417	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13418	*/
13419	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13420	{
13421	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13422	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13423	}
13424
13425
13426	/**
13427	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13428	*/
13429	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13430	{
13431	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13432	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13433	}
13434
13435
13436	/**
13437	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13438	*/
13439	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13440	{
13441	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13442
13443	/*
13444	* Get the state we need and update the exit history entry.
13445	*/
13446	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13447	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13448	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13449	AssertRCReturn(rc, rc);
13450
13451	VBOXSTRICTRC rcStrict;
13452	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13453	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13454	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13455	if (!pExitRec)
13456	{
13457	/*
13458	* Regular CPUID instruction execution.
13459	*/
13460	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13461	if (rcStrict == VINF_SUCCESS)
13462	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13463	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13464	{
13465	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13466	rcStrict = VINF_SUCCESS;
13467	}
13468	}
13469	else
13470	{
13471	/*
13472	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13473	*/
13474	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13475	AssertRCReturn(rc2, rc2);
13476
13477	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13478	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13479
13480	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13481	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13482
13483	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13484	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13485	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13486	}
13487	return rcStrict;
13488	}
13489
13490
13491	/**
13492	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13493	*/
13494	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13495	{
13496	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13497
13498	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13499	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13500	AssertRCReturn(rc, rc);
13501
13502	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13503	return VINF_EM_RAW_EMULATE_INSTR;
13504
13505	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
13506	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13507	}
13508
13509
13510	/**
13511	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13512	*/
13513	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13514	{
13515	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13516
13517	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13518	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13519	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13520	AssertRCReturn(rc, rc);
13521
13522	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13523	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13524	{
13525	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13526	we must reset offsetting on VM-entry. See @bugref{6634}. */
13527	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13528	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13529	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13530	}
13531	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13532	{
13533	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13534	rcStrict = VINF_SUCCESS;
13535	}
13536	return rcStrict;
13537	}
13538
13539
13540	/**
13541	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13542	*/
13543	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13544	{
13545	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13546
13547	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13548	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13549	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13550	AssertRCReturn(rc, rc);
13551
13552	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13553	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13554	{
13555	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13556	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13557	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13558	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13559	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13560	}
13561	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13562	{
13563	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13564	rcStrict = VINF_SUCCESS;
13565	}
13566	return rcStrict;
13567	}
13568
13569
13570	/**
13571	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13572	*/
13573	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13574	{
13575	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13576
13577	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13578	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13579	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13580	AssertRCReturn(rc, rc);
13581
13582	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13583	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13584	if (RT_LIKELY(rc == VINF_SUCCESS))
13585	{
13586	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13587	Assert(pVmxTransient->cbInstr == 2);
13588	}
13589	else
13590	{
13591	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13592	rc = VERR_EM_INTERPRETER;
13593	}
13594	return rc;
13595	}
13596
13597
13598	/**
13599	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13600	*/
13601	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13602	{
13603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13604
13605	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13606	if (EMAreHypercallInstructionsEnabled(pVCpu))
13607	{
13608	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13609	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13610	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13611	AssertRCReturn(rc, rc);
13612
13613	/* Perform the hypercall. */
13614	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13615	if (rcStrict == VINF_SUCCESS)
13616	{
13617	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13618	AssertRCReturn(rc, rc);
13619	}
13620	else
13621	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13622	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13623	\|\| RT_FAILURE(rcStrict));
13624
13625	/* If the hypercall changes anything other than guest's general-purpose registers,
13626	we would need to reload the guest changed bits here before VM-entry. */
13627	}
13628	else
13629	Log4Func(("Hypercalls not enabled\n"));
13630
13631	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13632	if (RT_FAILURE(rcStrict))
13633	{
13634	hmR0VmxSetPendingXcptUD(pVCpu);
13635	rcStrict = VINF_SUCCESS;
13636	}
13637
13638	return rcStrict;
13639	}
13640
13641
13642	/**
13643	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13644	*/
13645	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13646	{
13647	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13648	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13649
13650	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13651	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13652	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13653	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13654	AssertRCReturn(rc, rc);
13655
13656	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13657
13658	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13659	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13660	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13661	{
13662	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13663	rcStrict = VINF_SUCCESS;
13664	}
13665	else
13666	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
13667	VBOXSTRICTRC_VAL(rcStrict)));
13668	return rcStrict;
13669	}
13670
13671
13672	/**
13673	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13674	*/
13675	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13676	{
13677	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13678
13679	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13680	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
13681	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13682	AssertRCReturn(rc, rc);
13683
13684	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
13685	if (rcStrict == VINF_SUCCESS)
13686	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13687	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13688	{
13689	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13690	rcStrict = VINF_SUCCESS;
13691	}
13692
13693	return rcStrict;
13694	}
13695
13696
13697	/**
13698	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13699	*/
13700	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13701	{
13702	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13703
13704	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13705	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13706	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13707	AssertRCReturn(rc, rc);
13708
13709	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
13710	if (RT_SUCCESS(rcStrict))
13711	{
13712	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13713	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
13714	rcStrict = VINF_SUCCESS;
13715	}
13716
13717	return rcStrict;
13718	}
13719
13720
13721	/**
13722	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13723	* VM-exit.
13724	*/
13725	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13726	{
13727	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13728	return VINF_EM_RESET;
13729	}
13730
13731
13732	/**
13733	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13734	*/
13735	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13736	{
13737	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13738
13739	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13740	AssertRCReturn(rc, rc);
13741
13742	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13743	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13744	rc = VINF_SUCCESS;
13745	else
13746	rc = VINF_EM_HALT;
13747
13748	if (rc != VINF_SUCCESS)
13749	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13750	return rc;
13751	}
13752
13753
13754	/**
13755	* VM-exit handler for instructions that result in a \#UD exception delivered to
13756	* the guest.
13757	*/
13758	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13759	{
13760	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13761	hmR0VmxSetPendingXcptUD(pVCpu);
13762	return VINF_SUCCESS;
13763	}
13764
13765
13766	/**
13767	* VM-exit handler for expiry of the VMX-preemption timer.
13768	*/
13769	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13770	{
13771	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13772
13773	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13774	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13775
13776	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13777	PVM pVM = pVCpu->CTX_SUFF(pVM);
13778	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13779	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13780	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13781	}
13782
13783
13784	/**
13785	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13786	*/
13787	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13788	{
13789	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13790
13791	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13792	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13793	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13794	AssertRCReturn(rc, rc);
13795
13796	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13797	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13798	: HM_CHANGED_RAISED_XCPT_MASK);
13799
13800	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13801	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13802
13803	return rcStrict;
13804	}
13805
13806
13807	/**
13808	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13809	*/
13810	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13811	{
13812	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13813	/** @todo Use VM-exit instruction information. */
13814	return VERR_EM_INTERPRETER;
13815	}
13816
13817
13818	/**
13819	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
13820	* VM-exit.
13821	*/
13822	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13823	{
13824	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13825	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13826	AssertRCReturn(rc, rc);
13827
13828	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13829	if (RT_FAILURE(rc))
13830	return rc;
13831
13832	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13833	NOREF(uInvalidReason);
13834
13835	#ifdef VBOX_STRICT
13836	uint32_t fIntrState;
13837	RTHCUINTREG uHCReg;
13838	uint64_t u64Val;
13839	uint32_t u32Val;
13840	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13841	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13842	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13843	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13844	AssertRCReturn(rc, rc);
13845
13846	Log4(("uInvalidReason %u\n", uInvalidReason));
13847	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13848	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13849	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13850	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13851
13852	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13853	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13854	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13855	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13856	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13857	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13858	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13859	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13860	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13861	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13862	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13863	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13864
13865	hmR0DumpRegs(pVCpu);
13866	#endif
13867
13868	return VERR_VMX_INVALID_GUEST_STATE;
13869	}
13870
13871	/**
13872	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
13873	*/
13874	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13875	{
13876	/*
13877	* Cummulative notes of all recognized but unexpected VM-exits.
13878	*
13879	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
13880	* nested-paging is used.
13881	*
13882	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
13883	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
13884	* this function (and thereby stop VM execution) for handling such instructions.
13885	*
13886	*
13887	* VMX_EXIT_INIT_SIGNAL:
13888	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13889	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
13890	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
13891	*
13892	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
13893	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
13894	* See Intel spec. "23.8 Restrictions on VMX operation".
13895	*
13896	* VMX_EXIT_SIPI:
13897	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
13898	* activity state is used. We don't make use of it as our guests don't have direct
13899	* access to the host local APIC.
13900	*
13901	* See Intel spec. 25.3 "Other Causes of VM-exits".
13902	*
13903	* VMX_EXIT_IO_SMI:
13904	* VMX_EXIT_SMI:
13905	* This can only happen if we support dual-monitor treatment of SMI, which can be
13906	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
13907	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
13908	* VMX root mode or receive an SMI. If we get here, something funny is going on.
13909	*
13910	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13911	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13912	*
13913	* VMX_EXIT_ERR_MSR_LOAD:
13914	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
13915	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
13916	* execution.
13917	*
13918	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
13919	*
13920	* VMX_EXIT_ERR_MACHINE_CHECK:
13921	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
13922	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
13923	* #MC exception abort class exception is raised. We thus cannot assume a
13924	* reasonable chance of continuing any sort of execution and we bail.
13925	*
13926	* See Intel spec. 15.1 "Machine-check Architecture".
13927	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
13928	*
13929	* VMX_EXIT_PML_FULL:
13930	* VMX_EXIT_VIRTUALIZED_EOI:
13931	* VMX_EXIT_APIC_WRITE:
13932	* We do not currently support any of these features and thus they are all unexpected
13933	* VM-exits.
13934	*
13935	* VMX_EXIT_GDTR_IDTR_ACCESS:
13936	* VMX_EXIT_LDTR_TR_ACCESS:
13937	* VMX_EXIT_RDRAND:
13938	* VMX_EXIT_RSM:
13939	* VMX_EXIT_VMFUNC:
13940	* VMX_EXIT_ENCLS:
13941	* VMX_EXIT_RDSEED:
13942	* VMX_EXIT_XSAVES:
13943	* VMX_EXIT_XRSTORS:
13944	* VMX_EXIT_UMWAIT:
13945	* VMX_EXIT_TPAUSE:
13946	* These VM-exits are -not- caused unconditionally by execution of the corresponding
13947	* instruction. Any VM-exit for these instructions indicate a hardware problem,
13948	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
13949	*
13950	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
13951	*/
13952	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13953	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
13954	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13955	}
13956
13957
13958	/**
13959	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13960	*/
13961	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13962	{
13963	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13964
13965	/** @todo Optimize this: We currently drag in in the whole MSR state
13966	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13967	* MSRs required. That would require changes to IEM and possibly CPUM too.
13968	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13969	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13970	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13971	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13972	switch (idMsr)
13973	{
13974	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13975	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13976	}
13977
13978	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13979	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13980	AssertRCReturn(rc, rc);
13981
13982	Log4Func(("ecx=%#RX32\n", idMsr));
13983
13984	#ifdef VBOX_STRICT
13985	if (hmR0VmxIsProcCtlsSet(pVCpu, pVmxTransient, VMX_PROC_CTLS_USE_MSR_BITMAPS))
13986	{
13987	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13988	&& idMsr != MSR_K6_EFER)
13989	{
13990	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13991	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13992	}
13993	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13994	{
13995	Assert(pVmcsInfo->pvMsrBitmap);
13996	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13997	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13998	{
13999	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14000	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14001	}
14002	}
14003	}
14004	#endif
14005
14006	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
14007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14008	if (rcStrict == VINF_SUCCESS)
14009	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14010	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14011	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14012	{
14013	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14014	rcStrict = VINF_SUCCESS;
14015	}
14016	else
14017	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14018
14019	return rcStrict;
14020	}
14021
14022
14023	/**
14024	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14025	*/
14026	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14027	{
14028	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14029
14030	/** @todo Optimize this: We currently drag in in the whole MSR state
14031	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14032	* MSRs required. That would require changes to IEM and possibly CPUM too.
14033	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14034	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14035	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14036
14037	/*
14038	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14039	* Although we don't need to fetch the base as it will be overwritten shortly, while
14040	* loading guest-state we would also load the entire segment register including limit
14041	* and attributes and thus we need to load them here.
14042	*/
14043	switch (idMsr)
14044	{
14045	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14046	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14047	}
14048
14049	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14050	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14051	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14052	AssertRCReturn(rc, rc);
14053
14054	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14055
14056	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14057	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14058
14059	if (rcStrict == VINF_SUCCESS)
14060	{
14061	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14062
14063	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14064	if ( idMsr == MSR_IA32_APICBASE
14065	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14066	&& idMsr <= MSR_IA32_X2APIC_END))
14067	{
14068	/*
14069	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14070	* When full APIC register virtualization is implemented we'll have to make
14071	* sure APIC state is saved from the VMCS before IEM changes it.
14072	*/
14073	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14074	}
14075	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14076	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14077	else if (idMsr == MSR_K6_EFER)
14078	{
14079	/*
14080	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14081	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14082	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14083	*/
14084	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14085	}
14086
14087	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
14088	if (!hmR0VmxIsProcCtlsSet(pVCpu, pVmxTransient, VMX_PROC_CTLS_USE_MSR_BITMAPS))
14089	{
14090	switch (idMsr)
14091	{
14092	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14093	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14094	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14095	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14096	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14097	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14098	default:
14099	{
14100	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14101	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14102	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14103	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14104	break;
14105	}
14106	}
14107	}
14108	#ifdef VBOX_STRICT
14109	else
14110	{
14111	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14112	switch (idMsr)
14113	{
14114	case MSR_IA32_SYSENTER_CS:
14115	case MSR_IA32_SYSENTER_EIP:
14116	case MSR_IA32_SYSENTER_ESP:
14117	case MSR_K8_FS_BASE:
14118	case MSR_K8_GS_BASE:
14119	{
14120	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14121	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14122	}
14123
14124	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14125	default:
14126	{
14127	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14128	{
14129	/* EFER MSR writes are always intercepted. */
14130	if (idMsr != MSR_K6_EFER)
14131	{
14132	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14133	idMsr));
14134	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14135	}
14136	}
14137
14138	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14139	{
14140	Assert(pVmcsInfo->pvMsrBitmap);
14141	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14142	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14143	{
14144	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14145	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14146	}
14147	}
14148	break;
14149	}
14150	}
14151	}
14152	#endif /* VBOX_STRICT */
14153	}
14154	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14155	{
14156	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14157	rcStrict = VINF_SUCCESS;
14158	}
14159	else
14160	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14161
14162	return rcStrict;
14163	}
14164
14165
14166	/**
14167	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14168	*/
14169	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14170	{
14171	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14172
14173	/** @todo The guest has likely hit a contended spinlock. We might want to
14174	* poke a schedule different guest VCPU. */
14175	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14176	if (RT_SUCCESS(rc))
14177	return VINF_EM_RAW_INTERRUPT;
14178
14179	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14180	return rc;
14181	}
14182
14183
14184	/**
14185	* VM-exit handler for when the TPR value is lowered below the specified
14186	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14187	*/
14188	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14189	{
14190	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14191	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14192
14193	/*
14194	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14195	* We'll re-evaluate pending interrupts and inject them before the next VM
14196	* entry so we can just continue execution here.
14197	*/
14198	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14199	return VINF_SUCCESS;
14200	}
14201
14202
14203	/**
14204	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14205	* VM-exit.
14206	*
14207	* @retval VINF_SUCCESS when guest execution can continue.
14208	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14209	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
14210	* incompatible guest state for VMX execution (real-on-v86 case).
14211	*/
14212	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14213	{
14214	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14215	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14216
14217	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14218	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14219	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14220	AssertRCReturn(rc, rc);
14221
14222	VBOXSTRICTRC rcStrict;
14223	PVM pVM = pVCpu->CTX_SUFF(pVM);
14224	uint64_t const uExitQual = pVmxTransient->uExitQual;
14225	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14226	switch (uAccessType)
14227	{
14228	/*
14229	* MOV to CRx.
14230	*/
14231	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
14232	{
14233	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14234	AssertRCReturn(rc, rc);
14235
14236	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
14237	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14238	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14239	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14240
14241	/*
14242	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
14243	* - When nested paging isn't used.
14244	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
14245	* - We are executing in the VM debug loop.
14246	*/
14247	Assert( iCrReg != 3
14248	\|\| !pVM->hm.s.fNestedPaging
14249	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14250	\|\| pVCpu->hm.s.fUsingDebugLoop);
14251
14252	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
14253	Assert( iCrReg != 8
14254	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14255
14256	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14257	AssertMsg( rcStrict == VINF_SUCCESS
14258	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14259
14260	/*
14261	* This is a kludge for handling switches back to real mode when we try to use
14262	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14263	* deal with special selector values, so we have to return to ring-3 and run
14264	* there till the selector values are V86 mode compatible.
14265	*
14266	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14267	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
14268	* this function.
14269	*/
14270	if ( iCrReg == 0
14271	&& rcStrict == VINF_SUCCESS
14272	&& !pVM->hm.s.vmx.fUnrestrictedGuest
14273	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14274	&& (uOldCr0 & X86_CR0_PE)
14275	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
14276	{
14277	/** @todo Check selectors rather than returning all the time. */
14278	Assert(!pVmxTransient->fIsNestedGuest);
14279	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14280	rcStrict = VINF_EM_RESCHEDULE_REM;
14281	}
14282	break;
14283	}
14284
14285	/*
14286	* MOV from CRx.
14287	*/
14288	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
14289	{
14290	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14291	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14292
14293	/*
14294	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
14295	* - When nested paging isn't used.
14296	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
14297	* - We are executing in the VM debug loop.
14298	*/
14299	Assert( iCrReg != 3
14300	\|\| !pVM->hm.s.fNestedPaging
14301	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14302	\|\| pVCpu->hm.s.fUsingDebugLoop);
14303
14304	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14305	Assert( iCrReg != 8
14306	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14307
14308	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14309	break;
14310	}
14311
14312	/*
14313	* CLTS (Clear Task-Switch Flag in CR0).
14314	*/
14315	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
14316	{
14317	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
14318	break;
14319	}
14320
14321	/*
14322	* LMSW (Load Machine-Status Word into CR0).
14323	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
14324	*/
14325	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
14326	{
14327	RTGCPTR GCPtrEffDst;
14328	uint8_t const cbInstr = pVmxTransient->cbInstr;
14329	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
14330	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
14331	if (fMemOperand)
14332	{
14333	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14334	AssertRCReturn(rc, rc);
14335	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
14336	}
14337	else
14338	GCPtrEffDst = NIL_RTGCPTR;
14339	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
14340	break;
14341	}
14342
14343	default:
14344	{
14345	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
14346	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
14347	}
14348	}
14349
14350	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14351	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14352	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
14353
14354	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14355	NOREF(pVM);
14356	return rcStrict;
14357	}
14358
14359
14360	/**
14361	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14362	* VM-exit.
14363	*/
14364	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14365	{
14366	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14367	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14368
14369	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14370	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14371	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14372	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14373	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14374	\| CPUMCTX_EXTRN_EFER);
14375	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14376	AssertRCReturn(rc, rc);
14377
14378	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14379	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14380	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14381	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14382	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14383	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14384	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14385	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14386
14387	/*
14388	* Update exit history to see if this exit can be optimized.
14389	*/
14390	VBOXSTRICTRC rcStrict;
14391	PCEMEXITREC pExitRec = NULL;
14392	if ( !fGstStepping
14393	&& !fDbgStepping)
14394	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14395	!fIOString
14396	? !fIOWrite
14397	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14398	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14399	: !fIOWrite
14400	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14401	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14402	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14403	if (!pExitRec)
14404	{
14405	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14406	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14407
14408	uint32_t const cbValue = s_aIOSizes[uIOSize];
14409	uint32_t const cbInstr = pVmxTransient->cbInstr;
14410	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14411	PVM pVM = pVCpu->CTX_SUFF(pVM);
14412	if (fIOString)
14413	{
14414	/*
14415	* INS/OUTS - I/O String instruction.
14416	*
14417	* Use instruction-information if available, otherwise fall back on
14418	* interpreting the instruction.
14419	*/
14420	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14421	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14422	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14423	if (fInsOutsInfo)
14424	{
14425	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14426	AssertRCReturn(rc2, rc2);
14427	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14428	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14429	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14430	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14431	if (fIOWrite)
14432	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14433	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14434	else
14435	{
14436	/*
14437	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14438	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14439	* See Intel Instruction spec. for "INS".
14440	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14441	*/
14442	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14443	}
14444	}
14445	else
14446	rcStrict = IEMExecOne(pVCpu);
14447
14448	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14449	fUpdateRipAlready = true;
14450	}
14451	else
14452	{
14453	/*
14454	* IN/OUT - I/O instruction.
14455	*/
14456	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14457	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14458	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14459	if (fIOWrite)
14460	{
14461	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14463	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14464	&& !pCtx->eflags.Bits.u1TF)
14465	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14466	}
14467	else
14468	{
14469	uint32_t u32Result = 0;
14470	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14471	if (IOM_SUCCESS(rcStrict))
14472	{
14473	/* Save result of I/O IN instr. in AL/AX/EAX. */
14474	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14475	}
14476	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14477	&& !pCtx->eflags.Bits.u1TF)
14478	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14480	}
14481	}
14482
14483	if (IOM_SUCCESS(rcStrict))
14484	{
14485	if (!fUpdateRipAlready)
14486	{
14487	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14488	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14489	}
14490
14491	/*
14492	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14493	* while booting Fedora 17 64-bit guest.
14494	*
14495	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14496	*/
14497	if (fIOString)
14498	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14499
14500	/*
14501	* If any I/O breakpoints are armed, we need to check if one triggered
14502	* and take appropriate action.
14503	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14504	*/
14505	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14506	AssertRCReturn(rc, rc);
14507
14508	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14509	* execution engines about whether hyper BPs and such are pending. */
14510	uint32_t const uDr7 = pCtx->dr[7];
14511	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14512	&& X86_DR7_ANY_RW_IO(uDr7)
14513	&& (pCtx->cr4 & X86_CR4_DE))
14514	\|\| DBGFBpIsHwIoArmed(pVM)))
14515	{
14516	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14517
14518	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14519	VMMRZCallRing3Disable(pVCpu);
14520	HM_DISABLE_PREEMPT(pVCpu);
14521
14522	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14523
14524	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14525	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14526	{
14527	/* Raise #DB. */
14528	if (fIsGuestDbgActive)
14529	ASMSetDR6(pCtx->dr[6]);
14530	if (pCtx->dr[7] != uDr7)
14531	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14532
14533	hmR0VmxSetPendingXcptDB(pVCpu);
14534	}
14535	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14536	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14537	else if ( rcStrict2 != VINF_SUCCESS
14538	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14539	rcStrict = rcStrict2;
14540	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14541
14542	HM_RESTORE_PREEMPT();
14543	VMMRZCallRing3Enable(pVCpu);
14544	}
14545	}
14546
14547	#ifdef VBOX_STRICT
14548	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14549	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14550	Assert(!fIOWrite);
14551	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14552	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14553	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14554	Assert(fIOWrite);
14555	else
14556	{
14557	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14558	* statuses, that the VMM device and some others may return. See
14559	* IOM_SUCCESS() for guidance. */
14560	AssertMsg( RT_FAILURE(rcStrict)
14561	\|\| rcStrict == VINF_SUCCESS
14562	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14563	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14564	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14565	\|\| rcStrict == VINF_EM_RAW_TO_R3
14566	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14567	# endif
14568	}
14569	#endif
14570	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14571	}
14572	else
14573	{
14574	/*
14575	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14576	*/
14577	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14578	AssertRCReturn(rc2, rc2);
14579	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14580	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14581	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14582	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14583	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14584	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14585
14586	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14587	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14588
14589	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14590	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14591	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14592	}
14593	return rcStrict;
14594	}
14595
14596
14597	/**
14598	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14599	* VM-exit.
14600	*/
14601	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14602	{
14603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14604
14605	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14606	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14607	AssertRCReturn(rc, rc);
14608	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14609	{
14610	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14611	AssertRCReturn(rc, rc);
14612	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14613	{
14614	uint32_t uErrCode;
14615	RTGCUINTPTR GCPtrFaultAddress;
14616	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14617	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14618	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14619	if (fErrorCodeValid)
14620	{
14621	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14622	AssertRCReturn(rc, rc);
14623	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14624	}
14625	else
14626	uErrCode = 0;
14627
14628	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14629	&& uVector == X86_XCPT_PF)
14630	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14631	else
14632	GCPtrFaultAddress = 0;
14633
14634	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14635	AssertRCReturn(rc, rc);
14636
14637	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14638	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14639
14640	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14641	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14642	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14643	}
14644	}
14645
14646	/* Fall back to the interpreter to emulate the task-switch. */
14647	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14648	return VERR_EM_INTERPRETER;
14649	}
14650
14651
14652	/**
14653	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14654	*/
14655	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14656	{
14657	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14658
14659	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14660	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14661	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14662	AssertRCReturn(rc, rc);
14663	return VINF_EM_DBG_STEPPED;
14664	}
14665
14666
14667	/**
14668	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14669	*/
14670	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14671	{
14672	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14674
14675	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14676	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14677	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14678	{
14679	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14680	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14681	{
14682	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14683	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14684	}
14685	}
14686	else
14687	{
14688	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14689	rcStrict1 = VINF_SUCCESS;
14690	return rcStrict1;
14691	}
14692
14693	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14694	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14695	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14696	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14697	AssertRCReturn(rc, rc);
14698
14699	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14700	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14701	VBOXSTRICTRC rcStrict2;
14702	switch (uAccessType)
14703	{
14704	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14705	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14706	{
14707	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14708	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14709	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14710
14711	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14712	GCPhys &= PAGE_BASE_GC_MASK;
14713	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14714	PVM pVM = pVCpu->CTX_SUFF(pVM);
14715	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14716	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14717
14718	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14719	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14720	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14721	CPUMCTX2CORE(pCtx), GCPhys);
14722	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14723	if ( rcStrict2 == VINF_SUCCESS
14724	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14725	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14726	{
14727	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14728	\| HM_CHANGED_GUEST_APIC_TPR);
14729	rcStrict2 = VINF_SUCCESS;
14730	}
14731	break;
14732	}
14733
14734	default:
14735	Log4Func(("uAccessType=%#x\n", uAccessType));
14736	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14737	break;
14738	}
14739
14740	if (rcStrict2 != VINF_SUCCESS)
14741	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14742	return rcStrict2;
14743	}
14744
14745
14746	/**
14747	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14748	* VM-exit.
14749	*/
14750	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14751	{
14752	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14753
14754	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14755	if (pVmxTransient->fWasGuestDebugStateActive)
14756	{
14757	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14758	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14759	}
14760
14761	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14762	if ( !pVCpu->hm.s.fSingleInstruction
14763	&& !pVmxTransient->fWasHyperDebugStateActive)
14764	{
14765	Assert(!DBGFIsStepping(pVCpu));
14766	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14767
14768	/* Don't intercept MOV DRx any more. */
14769	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14770	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14771	AssertRCReturn(rc, rc);
14772
14773	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14774	VMMRZCallRing3Disable(pVCpu);
14775	HM_DISABLE_PREEMPT(pVCpu);
14776
14777	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14778	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14779	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14780
14781	HM_RESTORE_PREEMPT();
14782	VMMRZCallRing3Enable(pVCpu);
14783
14784	#ifdef VBOX_WITH_STATISTICS
14785	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14786	AssertRCReturn(rc, rc);
14787	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14788	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14789	else
14790	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14791	#endif
14792	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14793	return VINF_SUCCESS;
14794	}
14795
14796	/*
14797	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14798	* The EFER MSR is always up-to-date.
14799	* Update the segment registers and DR7 from the CPU.
14800	*/
14801	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14802	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14803	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14804	AssertRCReturn(rc, rc);
14805	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14806
14807	PVM pVM = pVCpu->CTX_SUFF(pVM);
14808	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14809	{
14810	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14811	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14812	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14813	if (RT_SUCCESS(rc))
14814	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14815	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14816	}
14817	else
14818	{
14819	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14820	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14821	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14822	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14823	}
14824
14825	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14826	if (RT_SUCCESS(rc))
14827	{
14828	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14829	AssertRCReturn(rc2, rc2);
14830	return VINF_SUCCESS;
14831	}
14832	return rc;
14833	}
14834
14835
14836	/**
14837	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14838	* Conditional VM-exit.
14839	*/
14840	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14841	{
14842	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14843	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14844
14845	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14846	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14847	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14848	{
14849	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14850	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14851	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14852	{
14853	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14854	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14855	}
14856	}
14857	else
14858	{
14859	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14860	rcStrict1 = VINF_SUCCESS;
14861	return rcStrict1;
14862	}
14863
14864	/*
14865	* Get sufficent state and update the exit history entry.
14866	*/
14867	RTGCPHYS GCPhys;
14868	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14869	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14870	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14871	AssertRCReturn(rc, rc);
14872
14873	VBOXSTRICTRC rcStrict;
14874	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14875	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14876	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14877	if (!pExitRec)
14878	{
14879	/*
14880	* If we succeed, resume guest execution.
14881	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14882	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14883	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14884	* weird case. See @bugref{6043}.
14885	*/
14886	PVM pVM = pVCpu->CTX_SUFF(pVM);
14887	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14888	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14889	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14890	if ( rcStrict == VINF_SUCCESS
14891	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14892	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14893	{
14894	/* Successfully handled MMIO operation. */
14895	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14896	\| HM_CHANGED_GUEST_APIC_TPR);
14897	rcStrict = VINF_SUCCESS;
14898	}
14899	}
14900	else
14901	{
14902	/*
14903	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14904	*/
14905	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14906	AssertRCReturn(rc2, rc2);
14907
14908	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14909	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14910
14911	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14912	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14913
14914	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14915	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14916	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14917	}
14918	return VBOXSTRICTRC_TODO(rcStrict);
14919	}
14920
14921
14922	/**
14923	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14924	* VM-exit.
14925	*/
14926	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14927	{
14928	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14929	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14930
14931	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14932	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14933	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14934	{
14935	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14936	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14937	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14938	}
14939	else
14940	{
14941	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14942	rcStrict1 = VINF_SUCCESS;
14943	return rcStrict1;
14944	}
14945
14946	RTGCPHYS GCPhys;
14947	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14948	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14949	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14950	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14951	AssertRCReturn(rc, rc);
14952
14953	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14954	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14955
14956	RTGCUINT uErrorCode = 0;
14957	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14958	uErrorCode \|= X86_TRAP_PF_ID;
14959	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14960	uErrorCode \|= X86_TRAP_PF_RW;
14961	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14962	uErrorCode \|= X86_TRAP_PF_P;
14963
14964	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14965
14966
14967	/* Handle the pagefault trap for the nested shadow table. */
14968	PVM pVM = pVCpu->CTX_SUFF(pVM);
14969	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14970
14971	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14972	pCtx->cs.Sel, pCtx->rip));
14973
14974	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14975	TRPMResetTrap(pVCpu);
14976
14977	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14978	if ( rcStrict2 == VINF_SUCCESS
14979	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14980	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14981	{
14982	/* Successfully synced our nested page tables. */
14983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14984	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14985	return VINF_SUCCESS;
14986	}
14987
14988	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14989	return rcStrict2;
14990	}
14991
14992	/** @} */
14993
14994	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14995	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14996	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14997
14998	/**
14999	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
15000	*/
15001	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15002	{
15003	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15004	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
15005
15006	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
15007	AssertRCReturn(rc, rc);
15008
15009	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
15010	{
15011	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
15012	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
15013
15014	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
15015	* provides VM-exit instruction length. If this causes problem later,
15016	* disassemble the instruction like it's done on AMD-V. */
15017	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15018	AssertRCReturn(rc2, rc2);
15019	return rc;
15020	}
15021
15022	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15023	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15024	return rc;
15025	}
15026
15027
15028	/**
15029	* VM-exit exception handler for \#BP (Breakpoint exception).
15030	*/
15031	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15032	{
15033	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15034	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
15035
15036	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15037	AssertRCReturn(rc, rc);
15038
15039	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15040	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15041	if (rc == VINF_EM_RAW_GUEST_TRAP)
15042	{
15043	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15044	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15045	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15046	AssertRCReturn(rc, rc);
15047
15048	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15049	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15050	}
15051
15052	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
15053	return rc;
15054	}
15055
15056
15057	/**
15058	* VM-exit exception handler for \#AC (alignment check exception).
15059	*/
15060	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15061	{
15062	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15063
15064	/*
15065	* Re-inject it. We'll detect any nesting before getting here.
15066	*/
15067	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15068	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15069	AssertRCReturn(rc, rc);
15070	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15071
15072	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15073	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15074	return VINF_SUCCESS;
15075	}
15076
15077
15078	/**
15079	* VM-exit exception handler for \#DB (Debug exception).
15080	*/
15081	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15082	{
15083	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15084	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
15085
15086	/*
15087	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
15088	*/
15089	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15090
15091	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
15092	uint64_t const uDR6 = X86_DR6_INIT_VAL
15093	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
15094	\| X86_DR6_BD \| X86_DR6_BS));
15095
15096	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15097	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
15098	Log6Func(("rc=%Rrc\n", rc));
15099	if (rc == VINF_EM_RAW_GUEST_TRAP)
15100	{
15101	/*
15102	* The exception was for the guest. Update DR6, DR7.GD and
15103	* IA32_DEBUGCTL.LBR before forwarding it.
15104	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
15105	*/
15106	VMMRZCallRing3Disable(pVCpu);
15107	HM_DISABLE_PREEMPT(pVCpu);
15108
15109	pCtx->dr[6] &= ~X86_DR6_B_MASK;
15110	pCtx->dr[6] \|= uDR6;
15111	if (CPUMIsGuestDebugStateActive(pVCpu))
15112	ASMSetDR6(pCtx->dr[6]);
15113
15114	HM_RESTORE_PREEMPT();
15115	VMMRZCallRing3Enable(pVCpu);
15116
15117	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
15118	AssertRCReturn(rc, rc);
15119
15120	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
15121	pCtx->dr[7] &= ~X86_DR7_GD;
15122
15123	/* Paranoia. */
15124	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
15125	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
15126
15127	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
15128	AssertRCReturn(rc, rc);
15129
15130	/*
15131	* Raise #DB in the guest.
15132	*
15133	* It is important to reflect exactly what the VM-exit gave us (preserving the
15134	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
15135	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
15136	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
15137	*
15138	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
15139	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
15140	*/
15141	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15142	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15143	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15144	AssertRCReturn(rc, rc);
15145	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15146	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15147	return VINF_SUCCESS;
15148	}
15149
15150	/*
15151	* Not a guest trap, must be a hypervisor related debug event then.
15152	* Update DR6 in case someone is interested in it.
15153	*/
15154	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
15155	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
15156	CPUMSetHyperDR6(pVCpu, uDR6);
15157
15158	return rc;
15159	}
15160
15161
15162	/**
15163	* Hacks its way around the lovely mesa driver's backdoor accesses.
15164	*
15165	* @sa hmR0SvmHandleMesaDrvGp.
15166	*/
15167	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15168	{
15169	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15170	RT_NOREF(pCtx);
15171
15172	/* For now we'll just skip the instruction. */
15173	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15174	}
15175
15176
15177	/**
15178	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15179	* backdoor logging w/o checking what it is running inside.
15180	*
15181	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15182	* backdoor port and magic numbers loaded in registers.
15183	*
15184	* @returns true if it is, false if it isn't.
15185	* @sa hmR0SvmIsMesaDrvGp.
15186	*/
15187	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15188	{
15189	/* 0xed: IN eAX,dx */
15190	uint8_t abInstr[1];
15191	if (pVmxTransient->cbInstr != sizeof(abInstr))
15192	return false;
15193
15194	/* Check that it is #GP(0). */
15195	if (pVmxTransient->uExitIntErrorCode != 0)
15196	return false;
15197
15198	/* Check magic and port. */
15199	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15200	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15201	if (pCtx->rax != UINT32_C(0x564d5868))
15202	return false;
15203	if (pCtx->dx != UINT32_C(0x5658))
15204	return false;
15205
15206	/* Flat ring-3 CS. */
15207	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15208	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15209	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15210	if (pCtx->cs.Attr.n.u2Dpl != 3)
15211	return false;
15212	if (pCtx->cs.u64Base != 0)
15213	return false;
15214
15215	/* Check opcode. */
15216	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15217	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15218	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15219	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15220	if (RT_FAILURE(rc))
15221	return false;
15222	if (abInstr[0] != 0xed)
15223	return false;
15224
15225	return true;
15226	}
15227
15228
15229	/**
15230	* VM-exit exception handler for \#GP (General-protection exception).
15231	*
15232	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15233	*/
15234	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15235	{
15236	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15237	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15238
15239	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15240	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15241	if (pVmcsInfo->RealMode.fRealOnV86Active)
15242	{ /* likely */ }
15243	else
15244	{
15245	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15246	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
15247	#endif
15248	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15249	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15250	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15251	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15252	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15253	AssertRCReturn(rc, rc);
15254	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15255	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15256
15257	if ( pVmxTransient->fIsNestedGuest
15258	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15259	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15260	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15261	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15262	else
15263	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15264	return rc;
15265	}
15266
15267	Assert(CPUMIsGuestInRealModeEx(pCtx));
15268	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15269	Assert(!pVmxTransient->fIsNestedGuest);
15270
15271	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15272	AssertRCReturn(rc, rc);
15273
15274	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15275	if (rcStrict == VINF_SUCCESS)
15276	{
15277	if (!CPUMIsGuestInRealModeEx(pCtx))
15278	{
15279	/*
15280	* The guest is no longer in real-mode, check if we can continue executing the
15281	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15282	*/
15283	pVmcsInfo->RealMode.fRealOnV86Active = false;
15284	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15285	{
15286	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15287	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15288	}
15289	else
15290	{
15291	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15292	rcStrict = VINF_EM_RESCHEDULE;
15293	}
15294	}
15295	else
15296	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15297	}
15298	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15299	{
15300	rcStrict = VINF_SUCCESS;
15301	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15302	}
15303	return VBOXSTRICTRC_VAL(rcStrict);
15304	}
15305
15306
15307	/**
15308	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15309	* the exception reported in the VMX transient structure back into the VM.
15310	*
15311	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15312	* up-to-date.
15313	*/
15314	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15315	{
15316	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15317	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15318	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15319	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
15320	("uVector=%#x u32XcptBitmap=%#X32\n",
15321	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15322	NOREF(pVmcsInfo);
15323	#endif
15324
15325	/*
15326	* Re-inject the exception into the guest. This cannot be a double-fault condition which
15327	* would have been handled while checking exits due to event delivery.
15328	*/
15329	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15330	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15331	AssertRCReturn(rc, rc);
15332	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15333
15334	#ifdef DEBUG_ramshankar
15335	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15336	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15337	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15338	#endif
15339
15340	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15341	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15342	return VINF_SUCCESS;
15343	}
15344
15345
15346	/**
15347	* VM-exit exception handler for \#PF (Page-fault exception).
15348	*/
15349	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15350	{
15351	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15352	PVM pVM = pVCpu->CTX_SUFF(pVM);
15353	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15354	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15355	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15356	AssertRCReturn(rc, rc);
15357
15358	if (!pVM->hm.s.fNestedPaging)
15359	{ /* likely */ }
15360	else
15361	{
15362	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15363	Assert(pVCpu->hm.s.fUsingDebugLoop);
15364	#endif
15365	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15366	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15367	{
15368	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15369	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15370	}
15371	else
15372	{
15373	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15374	hmR0VmxSetPendingXcptDF(pVCpu);
15375	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15376	}
15377	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15378	return rc;
15379	}
15380
15381	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15382	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15383	if (pVmxTransient->fVectoringPF)
15384	{
15385	Assert(pVCpu->hm.s.Event.fPending);
15386	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15387	}
15388
15389	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15390	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15391	AssertRCReturn(rc, rc);
15392
15393	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15394	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15395
15396	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15397	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15398
15399	Log4Func(("#PF: rc=%Rrc\n", rc));
15400	if (rc == VINF_SUCCESS)
15401	{
15402	/*
15403	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15404	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15405	*/
15406	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15407	TRPMResetTrap(pVCpu);
15408	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15409	return rc;
15410	}
15411
15412	if (rc == VINF_EM_RAW_GUEST_TRAP)
15413	{
15414	if (!pVmxTransient->fVectoringDoublePF)
15415	{
15416	/* It's a guest page fault and needs to be reflected to the guest. */
15417	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15418	TRPMResetTrap(pVCpu);
15419	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15420	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15421	uGstErrorCode, pVmxTransient->uExitQual);
15422	}
15423	else
15424	{
15425	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15426	TRPMResetTrap(pVCpu);
15427	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15428	hmR0VmxSetPendingXcptDF(pVCpu);
15429	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15430	}
15431
15432	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15433	return VINF_SUCCESS;
15434	}
15435
15436	TRPMResetTrap(pVCpu);
15437	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15438	return rc;
15439	}
15440
15441
15442	/**
15443	* VM-exit helper for LMSW.
15444	*/
15445	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw,
15446	RTGCPTR GCPtrEffDst)
15447	{
15448	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15449	AssertRCReturn(rc, rc);
15450
15451	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
15452	AssertMsg( rcStrict == VINF_SUCCESS
15453	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15454
15455	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15456	if (rcStrict == VINF_IEM_RAISED_XCPT)
15457	{
15458	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15459	rcStrict = VINF_SUCCESS;
15460	}
15461
15462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
15463	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15464	return rcStrict;
15465	}
15466
15467
15468	/**
15469	* VM-exit helper for CLTS.
15470	*/
15471	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
15472	{
15473	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15474	AssertRCReturn(rc, rc);
15475
15476	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
15477	AssertMsg( rcStrict == VINF_SUCCESS
15478	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15479
15480	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15481	if (rcStrict == VINF_IEM_RAISED_XCPT)
15482	{
15483	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15484	rcStrict = VINF_SUCCESS;
15485	}
15486
15487	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
15488	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15489	return rcStrict;
15490	}
15491
15492
15493	/**
15494	* VM-exit helper for MOV from CRx (CRx read).
15495	*/
15496	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15497	{
15498	Assert(iCrReg < 16);
15499	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
15500
15501	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15502	AssertRCReturn(rc, rc);
15503
15504	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
15505	AssertMsg( rcStrict == VINF_SUCCESS
15506	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15507
15508	if (iGReg == X86_GREG_xSP)
15509	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
15510	else
15511	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15512	#ifdef VBOX_WITH_STATISTICS
15513	switch (iCrReg)
15514	{
15515	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
15516	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
15517	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
15518	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
15519	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
15520	}
15521	#endif
15522	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
15523	return rcStrict;
15524	}
15525
15526
15527	/**
15528	* VM-exit helper for MOV to CRx (CRx write).
15529	*/
15530	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15531	{
15532	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15533	AssertRCReturn(rc, rc);
15534
15535	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
15536	AssertMsg( rcStrict == VINF_SUCCESS
15537	\|\| rcStrict == VINF_IEM_RAISED_XCPT
15538	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15539
15540	switch (iCrReg)
15541	{
15542	case 0:
15543	{
15544	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15545	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
15546	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
15547	break;
15548	}
15549
15550	case 2:
15551	{
15552	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
15553	/* Nothing to do here, CR2 it's not part of the VMCS. */
15554	break;
15555	}
15556
15557	case 3:
15558	{
15559	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
15560	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
15561	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
15562	break;
15563	}
15564
15565	case 4:
15566	{
15567	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
15568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
15569	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
15570	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
15571	break;
15572	}
15573
15574	case 8:
15575	{
15576	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
15577	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
15578	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
15579	break;
15580	}
15581
15582	default:
15583	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
15584	break;
15585	}
15586
15587	if (rcStrict == VINF_IEM_RAISED_XCPT)
15588	{
15589	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15590	rcStrict = VINF_SUCCESS;
15591	}
15592	return rcStrict;
15593	}
15594
15595
15596	/**
15597	* VM-exit helper for handling host NMIs.
15598	*/
15599	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu)
15600	{
15601	VMXDispatchHostNmi();
15602
15603	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
15604	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
15605	return VINF_SUCCESS;
15606	}
15607
15608
15609	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15610	/** @name VMX instruction handlers.
15611	* @{
15612	*/
15613	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15614	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15615	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15616
15617	/**
15618	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15619	*/
15620	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15621	{
15622	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15623
15624	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15625	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15626	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15627	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15628	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15629	AssertRCReturn(rc, rc);
15630
15631	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15632
15633	VMXVEXITINFO ExitInfo;
15634	RT_ZERO(ExitInfo);
15635	ExitInfo.uReason = pVmxTransient->uExitReason;
15636	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15637	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15638	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15639	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15640
15641	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15642	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15643	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15644	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15645	{
15646	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15647	rcStrict = VINF_SUCCESS;
15648	}
15649	return rcStrict;
15650	}
15651
15652
15653	/**
15654	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15655	*/
15656	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15657	{
15658	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15659
15660	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15661	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15662	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15663	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15664	AssertRCReturn(rc, rc);
15665
15666	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15667
15668	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15669	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15670	{
15671	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15672	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15673	}
15674	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15675	return rcStrict;
15676	}
15677
15678
15679	/**
15680	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15681	*/
15682	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15683	{
15684	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15685
15686	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15687	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15688	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15689	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15690	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15691	AssertRCReturn(rc, rc);
15692
15693	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15694
15695	VMXVEXITINFO ExitInfo;
15696	RT_ZERO(ExitInfo);
15697	ExitInfo.uReason = pVmxTransient->uExitReason;
15698	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15699	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15700	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15701	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15702
15703	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15704	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15705	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15706	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15707	{
15708	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15709	rcStrict = VINF_SUCCESS;
15710	}
15711	return rcStrict;
15712	}
15713
15714
15715	/**
15716	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15717	*/
15718	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15719	{
15720	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15721
15722	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15723	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15724	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15725	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15726	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15727	AssertRCReturn(rc, rc);
15728
15729	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15730
15731	VMXVEXITINFO ExitInfo;
15732	RT_ZERO(ExitInfo);
15733	ExitInfo.uReason = pVmxTransient->uExitReason;
15734	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15735	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15736	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15737	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15738
15739	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15740	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15741	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15742	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15743	{
15744	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15745	rcStrict = VINF_SUCCESS;
15746	}
15747	return rcStrict;
15748	}
15749
15750
15751	/**
15752	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
15753	*/
15754	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15755	{
15756	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15757
15758	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15759	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15760	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15761	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15762	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15763	AssertRCReturn(rc, rc);
15764
15765	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15766
15767	VMXVEXITINFO ExitInfo;
15768	RT_ZERO(ExitInfo);
15769	ExitInfo.uReason = pVmxTransient->uExitReason;
15770	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15771	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15772	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15773	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15774	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15775
15776	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15777	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15778	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15779	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15780	{
15781	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15782	rcStrict = VINF_SUCCESS;
15783	}
15784	return rcStrict;
15785	}
15786
15787
15788	/**
15789	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15790	*/
15791	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15792	{
15793	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15794
15795	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15796	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15797	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15798	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15799	AssertRCReturn(rc, rc);
15800
15801	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15802
15803	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15804	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15805	{
15806	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15807	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15808	}
15809	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15810	return rcStrict;
15811	}
15812
15813
15814	/**
15815	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
15816	*/
15817	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15818	{
15819	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15820
15821	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15822	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15823	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15824	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15825	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15826	AssertRCReturn(rc, rc);
15827
15828	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15829
15830	VMXVEXITINFO ExitInfo;
15831	RT_ZERO(ExitInfo);
15832	ExitInfo.uReason = pVmxTransient->uExitReason;
15833	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15834	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15835	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15836	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15837	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15838
15839	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15840	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15841	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15842	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15843	{
15844	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15845	rcStrict = VINF_SUCCESS;
15846	}
15847	return rcStrict;
15848	}
15849
15850
15851	/**
15852	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15853	*/
15854	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15855	{
15856	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15857
15858	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15859	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15860	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15861	AssertRCReturn(rc, rc);
15862
15863	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15864
15865	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15866	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15867	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15868	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15869	{
15870	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15871	rcStrict = VINF_SUCCESS;
15872	}
15873	return rcStrict;
15874	}
15875
15876
15877	/**
15878	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15879	*/
15880	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15881	{
15882	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15883
15884	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15885	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15886	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15887	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15888	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15889	AssertRCReturn(rc, rc);
15890
15891	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15892
15893	VMXVEXITINFO ExitInfo;
15894	RT_ZERO(ExitInfo);
15895	ExitInfo.uReason = pVmxTransient->uExitReason;
15896	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15897	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15898	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15899	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15900
15901	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15902	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15903	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15904	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15905	{
15906	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15907	rcStrict = VINF_SUCCESS;
15908	}
15909	return rcStrict;
15910	}
15911
15912
15913	/**
15914	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
15915	*/
15916	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15917	{
15918	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15919
15920	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15921	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15922	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15923	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15924	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15925	AssertRCReturn(rc, rc);
15926
15927	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15928
15929	VMXVEXITINFO ExitInfo;
15930	RT_ZERO(ExitInfo);
15931	ExitInfo.uReason = pVmxTransient->uExitReason;
15932	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15933	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15934	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15935	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15936
15937	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
15938	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15939	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15940	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15941	{
15942	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15943	rcStrict = VINF_SUCCESS;
15944	}
15945	return rcStrict;
15946	}
15947
15948	/** @} */
15949
15950	/** @name Nested-guest VM-exit handlers.
15951	* @{
15952	*/
15953	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15954	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15955	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15956
15957	/**
15958	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
15959	* Conditional VM-exit.
15960	*/
15961	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15962	{
15963	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15964
15965	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15966	AssertRCReturn(rc, rc);
15967
15968	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
15969	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
15970	uint32_t const uExtIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
15971
15972	/*
15973	* Make sure not to use stale/previous VM-exit instruction length since we read the
15974	* instruction length from the VMCS below only for software exceptions and privileged
15975	* software exceptions but we pass it for all exception VM-exits below.
15976	*/
15977	pVmxTransient->cbInstr = 0;
15978
15979	switch (uExtIntType)
15980	{
15981	/*
15982	* Physical NMIs:
15983	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the
15984	* host.
15985	*/
15986	case VMX_EXIT_INT_INFO_TYPE_NMI:
15987	return hmR0VmxExitHostNmi(pVCpu);
15988
15989	/*
15990	* Hardware exceptions,
15991	* Software exceptions,
15992	* Privileged software exceptions:
15993	* Figure out if the exception must be delivered to the guest or the nested-guest.
15994	*
15995	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
15996	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
15997	* length.
15998	*/
15999	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16000	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16001	{
16002	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16003	RT_FALL_THRU();
16004	}
16005	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16006	{
16007	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16008	AssertRCReturn(rc, rc);
16009
16010	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
16011	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
16012	pVmxTransient->uExitIntErrorCode);
16013	if (fIntercept)
16014	{
16015	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16016	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16017	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16018	AssertRCReturn(rc, rc);
16019
16020	VMXVEXITINFO ExitInfo;
16021	RT_ZERO(ExitInfo);
16022	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16023	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16024
16025	VMXVEXITEVENTINFO ExitEventInfo;
16026	RT_ZERO(ExitEventInfo);
16027	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16028	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16029	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16030	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16031
16032	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16033	}
16034
16035	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
16036	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16037
16038	/* If the guest hypervisor is not intercepting the exception, forward it to the guest. */
16039	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
16040	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
16041	return VINF_SUCCESS;
16042	}
16043
16044	/*
16045	* Software interrupts:
16046	* VM-exits cannot be caused by software interrupts.
16047	*
16048	* External interrupts:
16049	* This should only happen when "acknowledge external interrupts on VM-exit"
16050	* control is set. However, we don't set it when executing guests or
16051	* nested-guests. For nested-guests it is emulated while injecting interrupts into
16052	* the guest.
16053	*/
16054	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16055	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16056	default:
16057	{
16058	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16059	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16060	}
16061	}
16062	}
16063
16064
16065	/**
16066	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16067	* Unconditional VM-exit.
16068	*/
16069	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16070	{
16071	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16072	return IEMExecVmxVmexitTripleFault(pVCpu);
16073	}
16074
16075
16076	/**
16077	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16078	*/
16079	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16080	{
16081	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16082
16083	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16084	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16085	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16086	}
16087
16088
16089	/**
16090	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16091	*/
16092	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16093	{
16094	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16095
16096	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16097	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16098	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16099	}
16100
16101
16102	/**
16103	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16104	* Unconditional VM-exit.
16105	*/
16106	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16107	{
16108	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16109
16110	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16111	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16112	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16113	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16114	AssertRCReturn(rc, rc);
16115
16116	VMXVEXITINFO ExitInfo;
16117	RT_ZERO(ExitInfo);
16118	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16119	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16120
16121	VMXVEXITEVENTINFO ExitEventInfo;
16122	RT_ZERO(ExitInfo);
16123	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16124	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16125	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16126	}
16127
16128
16129	/**
16130	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16131	*/
16132	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16133	{
16134	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16135
16136	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16137	{
16138	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16139	AssertRCReturn(rc, rc);
16140	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16141	}
16142	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16143	}
16144
16145
16146	/**
16147	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16148	*/
16149	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16150	{
16151	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16152
16153	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16154	{
16155	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16156	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16157	AssertRCReturn(rc, rc);
16158
16159	VMXVEXITINFO ExitInfo;
16160	RT_ZERO(ExitInfo);
16161	ExitInfo.uReason = pVmxTransient->uExitReason;
16162	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16163	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16164	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16165	}
16166	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16167	}
16168
16169
16170	/**
16171	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16172	*/
16173	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16174	{
16175	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16176
16177	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16178	{
16179	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16180	AssertRCReturn(rc, rc);
16181	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16182	}
16183	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16184	}
16185
16186
16187	/**
16188	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16189	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16190	*/
16191	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16192	{
16193	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16194
16195	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16196	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16197
16198	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16199	AssertRCReturn(rc, rc);
16200
16201	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16202	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16203	uint64_t u64FieldEnc = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16204
16205	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16206	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16207	u64FieldEnc &= UINT64_C(0xffffffff);
16208
16209	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64FieldEnc))
16210	{
16211	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16212	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16213	AssertRCReturn(rc, rc);
16214
16215	VMXVEXITINFO ExitInfo;
16216	RT_ZERO(ExitInfo);
16217	ExitInfo.uReason = pVmxTransient->uExitReason;
16218	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16219	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16220	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16221	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16222	}
16223
16224	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16225	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16226	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16227	}
16228
16229
16230	/**
16231	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16232	*/
16233	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16234	{
16235	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16236
16237	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16238	{
16239	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16240	AssertRCReturn(rc, rc);
16241	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16242	}
16243
16244	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16245	}
16246
16247
16248	/**
16249	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16250	* Conditional VM-exit.
16251	*/
16252	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16253	{
16254	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16255
16256	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16257	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16258	AssertRCReturn(rc, rc);
16259
16260	VBOXSTRICTRC rcStrict;
16261	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16262	switch (uAccessType)
16263	{
16264	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16265	{
16266	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16267	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16268	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16269	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16270	if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX))
16271	{
16272	VMXVEXITINFO ExitInfo;
16273	RT_ZERO(ExitInfo);
16274	ExitInfo.uReason = pVmxTransient->uExitReason;
16275	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16276	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16277	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16278	}
16279	else
16280	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16281	break;
16282	}
16283
16284	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16285	{
16286	/*
16287	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16288	* CR2 reads do not cause a VM-exit.
16289	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16290	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16291	*/
16292	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16293	if ( iCrReg == 3
16294	\|\| iCrReg == 8)
16295	{
16296	static const uint32_t s_aCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16297	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16298	uint32_t const uIntercept = s_aCrXReadIntercepts[iCrReg];
16299	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16300	{
16301	VMXVEXITINFO ExitInfo;
16302	RT_ZERO(ExitInfo);
16303	ExitInfo.uReason = pVmxTransient->uExitReason;
16304	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16305	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16306	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16307	}
16308	else
16309	{
16310	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16311	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16312	}
16313	}
16314	else
16315	{
16316	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16317	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16318	}
16319	break;
16320	}
16321
16322	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16323	{
16324	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16325	Assert(pVmcsNstGst);
16326	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16327	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16328	if ( (uGstHostMask & X86_CR0_TS)
16329	&& (uReadShadow & X86_CR0_TS))
16330	{
16331	VMXVEXITINFO ExitInfo;
16332	RT_ZERO(ExitInfo);
16333	ExitInfo.uReason = pVmxTransient->uExitReason;
16334	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16335	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16336	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16337	}
16338	else
16339	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
16340	break;
16341	}
16342
16343	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16344	{
16345	RTGCPTR GCPtrEffDst;
16346	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16347	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16348	if (fMemOperand)
16349	{
16350	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16351	AssertRCReturn(rc, rc);
16352	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16353	}
16354	else
16355	GCPtrEffDst = NIL_RTGCPTR;
16356
16357	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16358	{
16359	VMXVEXITINFO ExitInfo;
16360	RT_ZERO(ExitInfo);
16361	ExitInfo.uReason = pVmxTransient->uExitReason;
16362	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16363	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16364	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16365	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16366	}
16367	else
16368	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
16369	break;
16370	}
16371
16372	default:
16373	{
16374	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16375	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16376	}
16377	}
16378
16379	if (rcStrict == VINF_IEM_RAISED_XCPT)
16380	{
16381	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16382	rcStrict = VINF_SUCCESS;
16383	}
16384	return rcStrict;
16385	}
16386
16387
16388	/**
16389	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16390	* Conditional VM-exit.
16391	*/
16392	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16393	{
16394	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16395
16396	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16397	{
16398	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16399	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16400	AssertRCReturn(rc, rc);
16401
16402	VMXVEXITINFO ExitInfo;
16403	RT_ZERO(ExitInfo);
16404	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16405	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16406	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16407	}
16408	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16409	}
16410
16411
16412	/**
16413	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16414	* Conditional VM-exit.
16415	*/
16416	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16417	{
16418	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16419
16420	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16421	AssertRCReturn(rc, rc);
16422
16423	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16424	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16425	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16426
16427	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16428	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16429	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16430	{
16431	/*
16432	* IN/OUT instruction:
16433	* - Provides VM-exit instruction length.
16434	*
16435	* INS/OUTS instruction:
16436	* - Provides VM-exit instruction length.
16437	* - Provides Guest-linear address.
16438	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16439	*/
16440	PVM pVM = pVCpu->CTX_SUFF(pVM);
16441	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16442	AssertRCReturn(rc, rc);
16443
16444	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16445	pVmxTransient->ExitInstrInfo.u = 0;
16446	pVmxTransient->uGuestLinearAddr = 0;
16447
16448	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16449	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16450	if (fIOString)
16451	{
16452	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16453	if (fVmxInsOutsInfo)
16454	{
16455	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16456	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16457	}
16458	}
16459	AssertRCReturn(rc, rc);
16460
16461	VMXVEXITINFO ExitInfo;
16462	RT_ZERO(ExitInfo);
16463	ExitInfo.uReason = pVmxTransient->uExitReason;
16464	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16465	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16466	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16467	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16468	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16469	}
16470	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16471	}
16472
16473
16474	/**
16475	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16476	*/
16477	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16478	{
16479	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16480
16481	uint32_t fMsrpm;
16482	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16483	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16484	else
16485	fMsrpm = VMXMSRPM_EXIT_RD;
16486
16487	if (fMsrpm & VMXMSRPM_EXIT_RD)
16488	{
16489	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16490	AssertRCReturn(rc, rc);
16491	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16492	}
16493	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16494	}
16495
16496
16497	/**
16498	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16499	*/
16500	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16501	{
16502	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16503
16504	uint32_t fMsrpm;
16505	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16506	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16507	else
16508	fMsrpm = VMXMSRPM_EXIT_WR;
16509
16510	if (fMsrpm & VMXMSRPM_EXIT_WR)
16511	{
16512	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16513	AssertRCReturn(rc, rc);
16514	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16515	}
16516	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16517	}
16518
16519
16520	/**
16521	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16522	*/
16523	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16524	{
16525	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16526
16527	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16528	{
16529	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16530	AssertRCReturn(rc, rc);
16531	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16532	}
16533	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16534	}
16535
16536
16537	/**
16538	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16539	* VM-exit.
16540	*/
16541	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16542	{
16543	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16544
16545	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16546	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16547	}
16548
16549
16550	/**
16551	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16552	*/
16553	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16554	{
16555	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16556
16557	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16558	{
16559	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16560	AssertRCReturn(rc, rc);
16561	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16562	}
16563	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16564	}
16565
16566
16567	/**
16568	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16569	*/
16570	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16571	{
16572	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16573
16574	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16575	* PAUSE when executing a nested-guest? If it does not, we would not need
16576	* to check for the intercepts here. Just call VM-exit... */
16577
16578	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16579	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16580	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16581	{
16582	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16583	AssertRCReturn(rc, rc);
16584	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16585	}
16586	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16587	}
16588
16589
16590	/**
16591	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16592	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16593	*/
16594	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16595	{
16596	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16597
16598	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16599	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16600	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16601	}
16602
16603
16604	/**
16605	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16606	* VM-exit.
16607	*/
16608	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16609	{
16610	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16611
16612	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16613	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16614	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16615	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16616	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16617	AssertRCReturn(rc, rc);
16618
16619	VMXVEXITINFO ExitInfo;
16620	RT_ZERO(ExitInfo);
16621	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16622	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16623
16624	VMXVEXITEVENTINFO ExitEventInfo;
16625	RT_ZERO(ExitInfo);
16626	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16627	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16628	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
16629	}
16630
16631
16632	/**
16633	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
16634	* Conditional VM-exit.
16635	*/
16636	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16637	{
16638	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16639
16640	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
16641	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16642	AssertRCReturn(rc, rc);
16643
16644	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16645	}
16646
16647
16648	/**
16649	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
16650	* Conditional VM-exit.
16651	*/
16652	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16653	{
16654	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16655
16656	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
16657	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16658	AssertRCReturn(rc, rc);
16659
16660	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16661	}
16662
16663
16664	/**
16665	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16666	*/
16667	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16668	{
16669	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16670
16671	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16672	{
16673	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16674	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16675	AssertRCReturn(rc, rc);
16676	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16677	}
16678	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16679	}
16680
16681
16682	/**
16683	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16684	*/
16685	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16686	{
16687	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16688
16689	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16690	{
16691	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16692	AssertRCReturn(rc, rc);
16693	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16694	}
16695	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16696	}
16697
16698
16699	/**
16700	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16701	*/
16702	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16703	{
16704	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16705
16706	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16707	{
16708	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
16709	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16710	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16711	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16712	AssertRCReturn(rc, rc);
16713
16714	VMXVEXITINFO ExitInfo;
16715	RT_ZERO(ExitInfo);
16716	ExitInfo.uReason = pVmxTransient->uExitReason;
16717	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16718	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16719	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16720	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16721	}
16722	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
16723	}
16724
16725
16726	/**
16727	* Nested-guest VM-exit handler for invalid-guest state
16728	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
16729	*/
16730	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16731	{
16732	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16733
16734	/*
16735	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
16736	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
16737	* Handle it like it's in an invalid guest state of the outer guest.
16738	*
16739	* When the fast path is implemented, this should be changed to cause the corresponding
16740	* nested-guest VM-exit.
16741	*/
16742	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
16743	}
16744
16745
16746	/**
16747	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
16748	* and only provide the instruction length.
16749	*
16750	* Unconditional VM-exit.
16751	*/
16752	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16753	{
16754	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16755
16756	#ifdef VBOX_STRICT
16757	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16758	switch (pVmxTransient->uExitReason)
16759	{
16760	case VMX_EXIT_ENCLS:
16761	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
16762	break;
16763
16764	case VMX_EXIT_VMFUNC:
16765	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
16766	break;
16767	}
16768	#endif
16769
16770	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16771	AssertRCReturn(rc, rc);
16772	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16773	}
16774
16775
16776	/**
16777	* Nested-guest VM-exit handler for instructions that provide instruction length as
16778	* well as more information.
16779	*
16780	* Unconditional VM-exit.
16781	*/
16782	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16783	{
16784	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16785
16786	#ifdef VBOX_STRICT
16787	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16788	switch (pVmxTransient->uExitReason)
16789	{
16790	case VMX_EXIT_GDTR_IDTR_ACCESS:
16791	case VMX_EXIT_LDTR_TR_ACCESS:
16792	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
16793	break;
16794
16795	case VMX_EXIT_RDRAND:
16796	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
16797	break;
16798
16799	case VMX_EXIT_RDSEED:
16800	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
16801	break;
16802
16803	case VMX_EXIT_XSAVES:
16804	case VMX_EXIT_XRSTORS:
16805	/** @todo NSTVMX: Verify XSS-bitmap. */
16806	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
16807	break;
16808
16809	case VMX_EXIT_UMWAIT:
16810	case VMX_EXIT_TPAUSE:
16811	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
16812	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
16813	break;
16814	}
16815	#endif
16816
16817	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16818	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16819	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16820	AssertRCReturn(rc, rc);
16821
16822	VMXVEXITINFO ExitInfo;
16823	RT_ZERO(ExitInfo);
16824	ExitInfo.uReason = pVmxTransient->uExitReason;
16825	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16826	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16827	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16828	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16829	}
16830
16831	/** @} */
16832
16833	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16834

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79185

Download in other formats: