HMSVMR0.cpp@ 87700

Last change on this file since 87700 was 87606, checked in by vboxsync, 4 years ago
VMM/HMVMX: Translate fMdsClearOnSched and fL1dFlushOnSched to world switcher flags too and use them in VMXR0Enter and VMXR0ThreadCtxCallback. Added missing MDS flushing to the latter. Moved the flushing up to the start of the functions. bugref:9453 bugref:9087
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 319.3 KB

Line
1	/* $Id: HMSVMR0.cpp 87606 2021-02-04 13:35:36Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#include "HMInternal.h"
36	#include <VBox/vmm/vmcc.h>
37	#include <VBox/err.h>
38	#include "HMSVMR0.h"
39	#include "dtrace/VBoxVMM.h"
40
41	#ifdef DEBUG_ramshankar
42	# define HMSVM_SYNC_FULL_GUEST_STATE
43	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
44	# define HMSVM_ALWAYS_TRAP_PF
45	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
46	#endif
47
48
49	/*********************************************************************************************************************************
50	* Defined Constants And Macros *
51	*********************************************************************************************************************************/
52	#ifdef VBOX_WITH_STATISTICS
53	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
54	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
55	if ((u64ExitCode) == SVM_EXIT_NPF) \
56	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
57	else \
58	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
59	} while (0)
60
61	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
62	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
63	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
64	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
65	if ((u64ExitCode) == SVM_EXIT_NPF) \
66	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
67	else \
68	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
69	} while (0)
70	# endif
71	#else
72	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
73	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
74	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
75	# endif
76	#endif /* !VBOX_WITH_STATISTICS */
77
78	/** If we decide to use a function table approach this can be useful to
79	* switch to a "static DECLCALLBACK(int)". */
80	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
81
82	/**
83	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
84	* guest using hardware-assisted SVM.
85	*
86	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
87	* are swapped and restored across the world-switch and also registers like
88	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
89	* \#VMEXIT.
90	*/
91	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
92	\| CPUMCTX_EXTRN_RFLAGS \
93	\| CPUMCTX_EXTRN_RAX \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_CR0 \
97	\| CPUMCTX_EXTRN_CR2 \
98	\| CPUMCTX_EXTRN_CR3 \
99	\| CPUMCTX_EXTRN_TABLE_MASK \
100	\| CPUMCTX_EXTRN_DR6 \
101	\| CPUMCTX_EXTRN_DR7 \
102	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
103	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
104	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
105	\| CPUMCTX_EXTRN_HWVIRT \
106	\| CPUMCTX_EXTRN_HM_SVM_MASK)
107
108	/**
109	* Subset of the guest-CPU state that is shared between the guest and host.
110	*/
111	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
112
113	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
114	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
115	do { \
116	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
117	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
118	} while (0)
119
120	/** Assert that the required state bits are fetched. */
121	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
122	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
123	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
124
125	/** Assert that preemption is disabled or covered by thread-context hooks. */
126	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
127	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
128
129	/** Assert that we haven't migrated CPUs when thread-context hooks are not
130	* used. */
131	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
132	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
133	("Illegal migration! Entered on CPU %u Current %u\n", \
134	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
135
136	/** Assert that we're not executing a nested-guest. */
137	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
138	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
139	#else
140	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
141	#endif
142
143	/** Assert that we're executing a nested-guest. */
144	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
145	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
146	#else
147	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
148	#endif
149
150	/** Macro for checking and returning from the using function for
151	* \#VMEXIT intercepts that maybe caused during delivering of another
152	* event in the guest. */
153	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
154	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
155	do \
156	{ \
157	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
158	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
159	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
160	else if ( rc == VINF_EM_RESET \
161	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
162	{ \
163	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
164	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
165	} \
166	else \
167	return rc; \
168	} while (0)
169	#else
170	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
171	do \
172	{ \
173	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
174	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
175	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
176	else \
177	return rc; \
178	} while (0)
179	#endif
180
181	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
182	* instruction that exited. */
183	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
184	do { \
185	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
186	(a_rc) = VINF_EM_DBG_STEPPED; \
187	} while (0)
188
189	/** Validate segment descriptor granularity bit. */
190	#ifdef VBOX_STRICT
191	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
192	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
193	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
194	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
195	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
196	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
197	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
198	#else
199	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
200	#endif
201
202	/**
203	* Exception bitmap mask for all contributory exceptions.
204	*
205	* Page fault is deliberately excluded here as it's conditional as to whether
206	* it's contributory or benign. Page faults are handled separately.
207	*/
208	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
209	\| RT_BIT(X86_XCPT_DE))
210
211	/**
212	* Mandatory/unconditional guest control intercepts.
213	*
214	* SMIs can and do happen in normal operation. We need not intercept them
215	* while executing the guest (or nested-guest).
216	*/
217	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
218	\| SVM_CTRL_INTERCEPT_NMI \
219	\| SVM_CTRL_INTERCEPT_INIT \
220	\| SVM_CTRL_INTERCEPT_RDPMC \
221	\| SVM_CTRL_INTERCEPT_CPUID \
222	\| SVM_CTRL_INTERCEPT_RSM \
223	\| SVM_CTRL_INTERCEPT_HLT \
224	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
225	\| SVM_CTRL_INTERCEPT_MSR_PROT \
226	\| SVM_CTRL_INTERCEPT_INVLPGA \
227	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
228	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
229	\| SVM_CTRL_INTERCEPT_VMRUN \
230	\| SVM_CTRL_INTERCEPT_SKINIT \
231	\| SVM_CTRL_INTERCEPT_WBINVD \
232	\| SVM_CTRL_INTERCEPT_MONITOR \
233	\| SVM_CTRL_INTERCEPT_MWAIT \
234	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
235	\| SVM_CTRL_INTERCEPT_XSETBV)
236
237	/** @name VMCB Clean Bits.
238	*
239	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
240	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
241	* memory.
242	*
243	* @{ */
244	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
245	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
246	/** I/O permission bitmap, MSR permission bitmap. */
247	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
248	/** ASID. */
249	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
250	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
251	V_INTR_VECTOR. */
252	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
253	/** Nested Paging: Nested CR3 (nCR3), PAT. */
254	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
255	/** Control registers (CR0, CR3, CR4, EFER). */
256	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
257	/** Debug registers (DR6, DR7). */
258	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
259	/** GDT, IDT limit and base. */
260	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
261	/** Segment register: CS, SS, DS, ES limit and base. */
262	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
263	/** CR2.*/
264	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
265	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
266	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
267	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
268	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
269	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
270	/** Mask of all valid VMCB Clean bits. */
271	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
272	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
273	\| HMSVM_VMCB_CLEAN_ASID \
274	\| HMSVM_VMCB_CLEAN_INT_CTRL \
275	\| HMSVM_VMCB_CLEAN_NP \
276	\| HMSVM_VMCB_CLEAN_CRX_EFER \
277	\| HMSVM_VMCB_CLEAN_DRX \
278	\| HMSVM_VMCB_CLEAN_DT \
279	\| HMSVM_VMCB_CLEAN_SEG \
280	\| HMSVM_VMCB_CLEAN_CR2 \
281	\| HMSVM_VMCB_CLEAN_LBR \
282	\| HMSVM_VMCB_CLEAN_AVIC)
283	/** @} */
284
285	/** @name SVM transient.
286	*
287	* A state structure for holding miscellaneous information across AMD-V
288	* VMRUN/\#VMEXIT operation, restored after the transition.
289	*
290	* @{ */
291	typedef struct SVMTRANSIENT
292	{
293	/** The host's rflags/eflags. */
294	RTCCUINTREG fEFlags;
295	/** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
296	uint64_t u64ExitCode;
297
298	/** The guest's TPR value used for TPR shadowing. */
299	uint8_t u8GuestTpr;
300	/** Alignment. */
301	uint8_t abAlignment0[7];
302
303	/** Pointer to the currently executing VMCB. */
304	PSVMVMCB pVmcb;
305
306	/** Whether we are currently executing a nested-guest. */
307	bool fIsNestedGuest;
308	/** Whether the guest debug state was active at the time of \#VMEXIT. */
309	bool fWasGuestDebugStateActive;
310	/** Whether the hyper debug state was active at the time of \#VMEXIT. */
311	bool fWasHyperDebugStateActive;
312	/** Whether the TSC offset mode needs to be updated. */
313	bool fUpdateTscOffsetting;
314	/** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
315	bool fRestoreTscAuxMsr;
316	/** Whether the \#VMEXIT was caused by a page-fault during delivery of a
317	* contributary exception or a page-fault. */
318	bool fVectoringDoublePF;
319	/** Whether the \#VMEXIT was caused by a page-fault during delivery of an
320	* external interrupt or NMI. */
321	bool fVectoringPF;
322	/** Padding. */
323	bool afPadding0;
324	} SVMTRANSIENT;
325	/** Pointer to SVM transient state. */
326	typedef SVMTRANSIENT *PSVMTRANSIENT;
327	/** Pointer to a const SVM transient state. */
328	typedef const SVMTRANSIENT *PCSVMTRANSIENT;
329
330	AssertCompileSizeAlignment(SVMTRANSIENT, sizeof(uint64_t));
331	AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
332	AssertCompileMemberAlignment(SVMTRANSIENT, pVmcb, sizeof(uint64_t));
333	/** @} */
334
335	/**
336	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
337	*/
338	typedef enum SVMMSREXITREAD
339	{
340	/** Reading this MSR causes a \#VMEXIT. */
341	SVMMSREXIT_INTERCEPT_READ = 0xb,
342	/** Reading this MSR does not cause a \#VMEXIT. */
343	SVMMSREXIT_PASSTHRU_READ
344	} SVMMSREXITREAD;
345
346	/**
347	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
348	*/
349	typedef enum SVMMSREXITWRITE
350	{
351	/** Writing to this MSR causes a \#VMEXIT. */
352	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
353	/** Writing to this MSR does not cause a \#VMEXIT. */
354	SVMMSREXIT_PASSTHRU_WRITE
355	} SVMMSREXITWRITE;
356
357	/**
358	* SVM \#VMEXIT handler.
359	*
360	* @returns Strict VBox status code.
361	* @param pVCpu The cross context virtual CPU structure.
362	* @param pSvmTransient Pointer to the SVM-transient structure.
363	*/
364	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
365
366
367	/*********************************************************************************************************************************
368	* Internal Functions *
369	*********************************************************************************************************************************/
370	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
371	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
372
373
374	/** @name \#VMEXIT handlers.
375	* @{
376	*/
377	static FNSVMEXITHANDLER hmR0SvmExitIntr;
378	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
379	static FNSVMEXITHANDLER hmR0SvmExitInvd;
380	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
381	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
382	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
383	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
384	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
385	static FNSVMEXITHANDLER hmR0SvmExitHlt;
386	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
387	static FNSVMEXITHANDLER hmR0SvmExitMwait;
388	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
389	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
390	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
391	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
392	static FNSVMEXITHANDLER hmR0SvmExitMsr;
393	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
394	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
395	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
396	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
397	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
398	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
399	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
400	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
401	static FNSVMEXITHANDLER hmR0SvmExitPause;
402	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
403	static FNSVMEXITHANDLER hmR0SvmExitIret;
404	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
405	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
406	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
407	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
408	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
409	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
410	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
411	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM)
412	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
413	#endif
414	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
415	static FNSVMEXITHANDLER hmR0SvmExitClgi;
416	static FNSVMEXITHANDLER hmR0SvmExitStgi;
417	static FNSVMEXITHANDLER hmR0SvmExitVmload;
418	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
419	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
420	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
421	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
422	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
423	#endif
424	/** @} */
425
426	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
427	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
428	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
429	#endif
430
431
432	/*********************************************************************************************************************************
433	* Global Variables *
434	*********************************************************************************************************************************/
435	/** Ring-0 memory object for the IO bitmap. */
436	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
437	/** Physical address of the IO bitmap. */
438	static RTHCPHYS g_HCPhysIOBitmap;
439	/** Pointer to the IO bitmap. */
440	static R0PTRTYPE(void *) g_pvIOBitmap;
441
442	#ifdef VBOX_STRICT
443	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
444	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
445	# define HMSVM_LOG_CS RT_BIT_32(2)
446	# define HMSVM_LOG_SS RT_BIT_32(3)
447	# define HMSVM_LOG_FS RT_BIT_32(4)
448	# define HMSVM_LOG_GS RT_BIT_32(5)
449	# define HMSVM_LOG_LBR RT_BIT_32(6)
450	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
451	\| HMSVM_LOG_CR_REGS \
452	\| HMSVM_LOG_CS \
453	\| HMSVM_LOG_SS \
454	\| HMSVM_LOG_FS \
455	\| HMSVM_LOG_GS \
456	\| HMSVM_LOG_LBR)
457
458	/**
459	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
460	*
461	* @param pVCpu The cross context virtual CPU structure.
462	* @param pVmcb Pointer to the VM control block.
463	* @param pszPrefix Log prefix.
464	* @param fFlags Log flags, see HMSVM_LOG_XXX.
465	* @param uVerbose The verbosity level, currently unused.
466	*/
467	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
468	{
469	RT_NOREF2(pVCpu, uVerbose);
470	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
471
472	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
473	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
474
475	if (fFlags & HMSVM_LOG_RBP_RSP)
476	{
477	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
478	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
479	}
480
481	if (fFlags & HMSVM_LOG_CR_REGS)
482	{
483	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
484	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
485	}
486
487	if (fFlags & HMSVM_LOG_CS)
488	{
489	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
490	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
491	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
492	}
493	if (fFlags & HMSVM_LOG_SS)
494	{
495	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
496	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
497	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
498	}
499	if (fFlags & HMSVM_LOG_FS)
500	{
501	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
502	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
503	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
504	}
505	if (fFlags & HMSVM_LOG_GS)
506	{
507	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
508	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
509	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
510	}
511
512	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
513	if (fFlags & HMSVM_LOG_LBR)
514	{
515	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
516	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
517	}
518	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
519	}
520	#endif /* VBOX_STRICT */
521
522
523	/**
524	* Sets up and activates AMD-V on the current CPU.
525	*
526	* @returns VBox status code.
527	* @param pHostCpu The HM physical-CPU structure.
528	* @param pVM The cross context VM structure. Can be
529	* NULL after a resume!
530	* @param pvCpuPage Pointer to the global CPU page.
531	* @param HCPhysCpuPage Physical address of the global CPU page.
532	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
533	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
534	* unused).
535	*/
536	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
537	PCSUPHWVIRTMSRS pHwvirtMsrs)
538	{
539	Assert(!fEnabledByHost);
540	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
541	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
542	Assert(pvCpuPage); NOREF(pvCpuPage);
543	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
544
545	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
546
547	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
548	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
549
550	/*
551	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
552	*/
553	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
554	if (u64HostEfer & MSR_K6_EFER_SVME)
555	{
556	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
557	if ( pVM
558	&& pVM->hm.s.svm.fIgnoreInUseError)
559	pHostCpu->fIgnoreAMDVInUseError = true;
560
561	if (!pHostCpu->fIgnoreAMDVInUseError)
562	{
563	ASMSetFlags(fEFlags);
564	return VERR_SVM_IN_USE;
565	}
566	}
567
568	/* Turn on AMD-V in the EFER MSR. */
569	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
570
571	/* Write the physical page address where the CPU will store the host state while executing the VM. */
572	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
573
574	/* Restore interrupts. */
575	ASMSetFlags(fEFlags);
576
577	/*
578	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
579	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
580	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
581	* entirely with before executing any guest code.
582	*/
583	pHostCpu->fFlushAsidBeforeUse = true;
584
585	/*
586	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
587	*/
588	++pHostCpu->cTlbFlushes;
589
590	return VINF_SUCCESS;
591	}
592
593
594	/**
595	* Deactivates AMD-V on the current CPU.
596	*
597	* @returns VBox status code.
598	* @param pHostCpu The HM physical-CPU structure.
599	* @param pvCpuPage Pointer to the global CPU page.
600	* @param HCPhysCpuPage Physical address of the global CPU page.
601	*/
602	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
603	{
604	RT_NOREF1(pHostCpu);
605	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
606	AssertReturn( HCPhysCpuPage
607	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
608	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
609
610	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
611	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
612
613	/* Turn off AMD-V in the EFER MSR. */
614	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
615	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
616
617	/* Invalidate host state physical address. */
618	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
619
620	/* Restore interrupts. */
621	ASMSetFlags(fEFlags);
622
623	return VINF_SUCCESS;
624	}
625
626
627	/**
628	* Does global AMD-V initialization (called during module initialization).
629	*
630	* @returns VBox status code.
631	*/
632	VMMR0DECL(int) SVMR0GlobalInit(void)
633	{
634	/*
635	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
636	* intercept all IO accesses, it's done once globally here instead of per-VM.
637	*/
638	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
639	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
640	if (RT_FAILURE(rc))
641	return rc;
642
643	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
644	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
645
646	/* Set all bits to intercept all IO accesses. */
647	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
648
649	return VINF_SUCCESS;
650	}
651
652
653	/**
654	* Does global AMD-V termination (called during module termination).
655	*/
656	VMMR0DECL(void) SVMR0GlobalTerm(void)
657	{
658	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
659	{
660	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
661	g_pvIOBitmap = NULL;
662	g_HCPhysIOBitmap = 0;
663	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
664	}
665	}
666
667
668	/**
669	* Frees any allocated per-VCPU structures for a VM.
670	*
671	* @param pVM The cross context VM structure.
672	*/
673	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
674	{
675	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
676	{
677	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
678	AssertPtr(pVCpu);
679
680	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
681	{
682	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
683	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
684	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
685	}
686
687	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
688	{
689	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
690	pVCpu->hmr0.s.svm.pVmcb = NULL;
691	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
692	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
693	}
694
695	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
696	{
697	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
698	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
699	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
700	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
701	}
702	}
703	}
704
705
706	/**
707	* Sets pfnVMRun to the best suited variant.
708	*
709	* This must be called whenever anything changes relative to the SVMR0VMRun
710	* variant selection:
711	* - pVCpu->hm.s.fLoadSaveGuestXcr0
712	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
713	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
714	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
715	* - Perhaps: CPUMCTX.fXStateMask (windows only)
716	*
717	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
718	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
719	*/
720	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
721	{
722	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
723	{
724	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
725	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
726	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
727	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
728	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
729	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
730	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
731	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
732	};
733	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
734	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
735	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
736	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
737	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
738	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
739	}
740
741
742	/**
743	* Selector FNHMSVMVMRUN implementation.
744	*/
745	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
746	{
747	hmR0SvmUpdateVmRunFunction(pVCpu);
748	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
749	}
750
751
752	/**
753	* Does per-VM AMD-V initialization.
754	*
755	* @returns VBox status code.
756	* @param pVM The cross context VM structure.
757	*/
758	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
759	{
760	int rc = VERR_INTERNAL_ERROR_5;
761
762	/*
763	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
764	*/
765	uint32_t u32Family;
766	uint32_t u32Model;
767	uint32_t u32Stepping;
768	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
769	{
770	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
771	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
772	}
773
774	/*
775	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
776	*/
777	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
778	{
779	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
780	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
781	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
782	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
783	}
784
785	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
786	{
787	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
788
789	/*
790	* Initialize the hardware-assisted SVM guest-execution handler.
791	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
792	*/
793	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
794
795	/*
796	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
797	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
798	*/
799	/** @todo Does this need to be below 4G? */
800	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
801	if (RT_FAILURE(rc))
802	goto failure_cleanup;
803
804	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
805	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
806	Assert(pVCpu->hmr0.s.svm.HCPhysVmcbHost < _4G);
807	ASMMemZeroPage(pvVmcbHost);
808
809	/*
810	* Allocate one page for the guest-state VMCB.
811	*/
812	/** @todo Does this need to be below 4G? */
813	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
814	if (RT_FAILURE(rc))
815	goto failure_cleanup;
816
817	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
818	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
819	Assert(pVCpu->hmr0.s.svm.HCPhysVmcb < _4G);
820	ASMMemZeroPage(pVCpu->hmr0.s.svm.pVmcb);
821
822	/*
823	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
824	* SVM to not require one.
825	*/
826	/** @todo Does this need to be below 4G? */
827	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
828	false /* fExecutable */);
829	if (RT_FAILURE(rc))
830	goto failure_cleanup;
831
832	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
833	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
834	/* Set all bits to intercept all MSR accesses (changed later on). */
835	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
836	}
837
838	return VINF_SUCCESS;
839
840	failure_cleanup:
841	hmR0SvmFreeStructs(pVM);
842	return rc;
843	}
844
845
846	/**
847	* Does per-VM AMD-V termination.
848	*
849	* @returns VBox status code.
850	* @param pVM The cross context VM structure.
851	*/
852	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
853	{
854	hmR0SvmFreeStructs(pVM);
855	return VINF_SUCCESS;
856	}
857
858
859	/**
860	* Returns whether the VMCB Clean Bits feature is supported.
861	*
862	* @returns @c true if supported, @c false otherwise.
863	* @param pVCpu The cross context virtual CPU structure.
864	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
865	*/
866	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
867	{
868	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
869	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
870	if (!fIsNestedGuest)
871	return fHostVmcbCleanBits;
872	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
873	}
874
875
876	/**
877	* Returns whether the decode assists feature is supported.
878	*
879	* @returns @c true if supported, @c false otherwise.
880	* @param pVCpu The cross context virtual CPU structure.
881	*/
882	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
883	{
884	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
885	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
886	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
887	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
888	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
889	#endif
890	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
891	}
892
893
894	/**
895	* Returns whether the NRIP_SAVE feature is supported.
896	*
897	* @returns @c true if supported, @c false otherwise.
898	* @param pVCpu The cross context virtual CPU structure.
899	*/
900	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
901	{
902	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
903	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
904	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
905	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
906	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
907	#endif
908	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
909	}
910
911
912	/**
913	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
914	*
915	* @param pVCpu The cross context virtual CPU structure.
916	* @param pbMsrBitmap Pointer to the MSR bitmap.
917	* @param idMsr The MSR for which the permissions are being set.
918	* @param enmRead MSR read permissions.
919	* @param enmWrite MSR write permissions.
920	*
921	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
922	* caller needs to take care of this.
923	*/
924	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
925	SVMMSREXITWRITE enmWrite)
926	{
927	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
928	uint16_t offMsrpm;
929	uint8_t uMsrpmBit;
930	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
931	AssertRC(rc);
932
933	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
934	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
935
936	pbMsrBitmap += offMsrpm;
937	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
938	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
939	else
940	{
941	if (!fInNestedGuestMode)
942	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
943	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
944	else
945	{
946	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
947	uint8_t const pbNstGstMsrBitmap = (uint8_t )pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
948	pbNstGstMsrBitmap += offMsrpm;
949	if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit)))
950	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
951	else
952	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
953	}
954	#endif
955	}
956
957	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
958	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
959	else
960	{
961	if (!fInNestedGuestMode)
962	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
963	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
964	else
965	{
966	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
967	uint8_t const pbNstGstMsrBitmap = (uint8_t )pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
968	pbNstGstMsrBitmap += offMsrpm;
969	if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit + 1)))
970	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
971	else
972	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
973	}
974	#endif
975	}
976	}
977
978
979	/**
980	* Sets up AMD-V for the specified VM.
981	* This function is only called once per-VM during initalization.
982	*
983	* @returns VBox status code.
984	* @param pVM The cross context VM structure.
985	*/
986	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
987	{
988	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
989	AssertReturn(pVM, VERR_INVALID_PARAMETER);
990
991	/*
992	* Validate and copy over some parameters.
993	*/
994	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
995	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
996	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
997	pVM->hmr0.s.fNestedPaging = fNestedPaging;
998	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
999
1000	/*
1001	* Determin some configuration parameters.
1002	*/
1003	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
1004	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
1005	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
1006
1007	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
1008	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
1009
1010	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1011	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
1012	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
1013
1014	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
1015	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
1016	#endif
1017
1018	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
1019	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
1020	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
1021	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
1022
1023	/* Always trap #AC for reasons of security. */
1024	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
1025
1026	/* Always trap #DB for reasons of security. */
1027	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
1028
1029	/* Trap exceptions unconditionally (debug purposes). */
1030	#ifdef HMSVM_ALWAYS_TRAP_PF
1031	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
1032	#endif
1033	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1034	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
1035	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
1036	\| RT_BIT_32(X86_XCPT_DE)
1037	\| RT_BIT_32(X86_XCPT_NM)
1038	\| RT_BIT_32(X86_XCPT_UD)
1039	\| RT_BIT_32(X86_XCPT_NP)
1040	\| RT_BIT_32(X86_XCPT_SS)
1041	\| RT_BIT_32(X86_XCPT_GP)
1042	\| RT_BIT_32(X86_XCPT_PF)
1043	\| RT_BIT_32(X86_XCPT_MF)
1044	;
1045	#endif
1046
1047	/* Apply the exceptions intercepts needed by the GIM provider. */
1048	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1049	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1050
1051	/* The mesa 3d driver hack needs #GP. */
1052	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1053	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1054
1055	/* Set up unconditional intercepts and conditions. */
1056	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1057	\| SVM_CTRL_INTERCEPT_VMMCALL
1058	\| SVM_CTRL_INTERCEPT_VMSAVE
1059	\| SVM_CTRL_INTERCEPT_VMLOAD
1060	\| SVM_CTRL_INTERCEPT_CLGI
1061	\| SVM_CTRL_INTERCEPT_STGI;
1062
1063	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1064	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1065	#endif
1066
1067	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1068	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1069	{
1070	/* Virtualized VMSAVE/VMLOAD. */
1071	if (fUseVirtVmsaveVmload)
1072	{
1073	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1074	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1075	\| SVM_CTRL_INTERCEPT_VMLOAD);
1076	}
1077	else
1078	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1079
1080	/* Virtual GIF. */
1081	if (fUseVGif)
1082	{
1083	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1084	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1085	\| SVM_CTRL_INTERCEPT_STGI);
1086	}
1087	else
1088	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1089	}
1090	else
1091	#endif
1092	{
1093	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1094	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1095	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1096	}
1097
1098	/* CR4 writes must always be intercepted for tracking PGM mode changes. */
1099	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1100
1101	/* Intercept all DRx reads and writes by default. Changed later on. */
1102	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1103	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1104
1105	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1106	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1107
1108	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1109	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1110	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1111
1112	/* Set the IO permission bitmap physical addresses. */
1113	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1114
1115	/* LBR virtualization. */
1116	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1117
1118	/* The host ASID MBZ, for the guest start with 1. */
1119	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1120
1121	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1122	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1123
1124	/* Without Nested Paging, we need additionally intercepts. */
1125	if (!fNestedPaging)
1126	{
1127	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1128	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1129	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1130
1131	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1132	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1133	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1134
1135	/* Page faults must be intercepted to implement shadow paging. */
1136	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1137	}
1138
1139	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1140	if (fUsePauseFilter)
1141	{
1142	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1143	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1144	if (fPauseFilterThreshold)
1145	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1146	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1147	}
1148
1149	/*
1150	* Setup the MSR permission bitmap.
1151	* The following MSRs are saved/restored automatically during the world-switch.
1152	* Don't intercept guest read/write accesses to these MSRs.
1153	*/
1154	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1155	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1156	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1157	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1158	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1159	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1160	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1161	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1162	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1163	{
1164	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1165	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1166	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1167	}
1168	else
1169	{
1170	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1171	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1172	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1173	}
1174	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1175
1176	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1177	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1178
1179	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1180	{
1181	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1182	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1183	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1184	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1185
1186	/* Copy the VMCB control area. */
1187	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1188
1189	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1190	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1191	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1192	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1193
1194	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1195	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1196
1197	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1198	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1199	}
1200
1201	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1202	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1203	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1204	#else
1205	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1206	#endif
1207	return VINF_SUCCESS;
1208	}
1209
1210
1211	/**
1212	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1213	*
1214	* @returns Pointer to the current context VMCB.
1215	* @param pVCpu The cross context virtual CPU structure.
1216	*/
1217	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1218	{
1219	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1220	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1221	return pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
1222	#endif
1223	return pVCpu->hmr0.s.svm.pVmcb;
1224	}
1225
1226
1227	/**
1228	* Gets a pointer to the nested-guest VMCB cache.
1229	*
1230	* @returns Pointer to the nested-guest VMCB cache.
1231	* @param pVCpu The cross context virtual CPU structure.
1232	*/
1233	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1234	{
1235	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1236	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1237	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1238	#else
1239	RT_NOREF(pVCpu);
1240	return NULL;
1241	#endif
1242	}
1243
1244
1245	/**
1246	* Invalidates a guest page by guest virtual address.
1247	*
1248	* @returns VBox status code.
1249	* @param pVCpu The cross context virtual CPU structure.
1250	* @param GCVirt Guest virtual address of the page to invalidate.
1251	*/
1252	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1253	{
1254	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1255
1256	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1257
1258	/* Skip it if a TLB flush is already pending. */
1259	if (!fFlushPending)
1260	{
1261	Log4Func(("%#RGv\n", GCVirt));
1262
1263	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1264	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1265
1266	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1267	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1268	}
1269	return VINF_SUCCESS;
1270	}
1271
1272
1273	/**
1274	* Flushes the appropriate tagged-TLB entries.
1275	*
1276	* @param pHostCpu The HM physical-CPU structure.
1277	* @param pVCpu The cross context virtual CPU structure.
1278	* @param pVmcb Pointer to the VM control block.
1279	*/
1280	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1281	{
1282	/*
1283	* Force a TLB flush for the first world switch if the current CPU differs from the one
1284	* we ran on last. This can happen both for start & resume due to long jumps back to
1285	* ring-3.
1286	*
1287	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1288	* correlation between it and the physical CPU.
1289	*
1290	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1291	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1292	*/
1293	bool fNewAsid = false;
1294	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1295	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1296	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1297	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1298	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1299	#endif
1300	)
1301	{
1302	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1303	pVCpu->hmr0.s.fForceTLBFlush = true;
1304	fNewAsid = true;
1305	}
1306
1307	/* Set TLB flush state as checked until we return from the world switch. */
1308	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1309
1310	/* Check for explicit TLB flushes. */
1311	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1312	{
1313	pVCpu->hmr0.s.fForceTLBFlush = true;
1314	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1315	}
1316
1317	/*
1318	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1319	* This Host CPU requirement takes precedence.
1320	*/
1321	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1322	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1323	{
1324	pHostCpu->uCurrentAsid = 1;
1325	pVCpu->hmr0.s.uCurrentAsid = 1;
1326	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1327	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1328	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1329
1330	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1331	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1332	}
1333	else
1334	{
1335	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1336	if (pVCpu->hmr0.s.fForceTLBFlush)
1337	{
1338	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1339	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1340
1341	if (fNewAsid)
1342	{
1343	++pHostCpu->uCurrentAsid;
1344
1345	bool fHitASIDLimit = false;
1346	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1347	{
1348	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1349	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1350	fHitASIDLimit = true;
1351	}
1352
1353	if ( fHitASIDLimit
1354	\|\| pHostCpu->fFlushAsidBeforeUse)
1355	{
1356	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1357	pHostCpu->fFlushAsidBeforeUse = false;
1358	}
1359
1360	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1361	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1362	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1363	}
1364	else
1365	{
1366	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1367	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1368	else
1369	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1370	}
1371
1372	pVCpu->hmr0.s.fForceTLBFlush = false;
1373	}
1374	}
1375
1376	/* Update VMCB with the ASID. */
1377	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1378	{
1379	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1380	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1381	}
1382
1383	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1384	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1385	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1386	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1387	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1388	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1389	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1390	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1391
1392	#ifdef VBOX_WITH_STATISTICS
1393	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1394	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1395	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1396	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1397	{
1398	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1399	}
1400	else
1401	{
1402	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1403	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1404	}
1405	#endif
1406	}
1407
1408
1409	/**
1410	* Sets an exception intercept in the specified VMCB.
1411	*
1412	* @param pVmcb Pointer to the VM control block.
1413	* @param uXcpt The exception (X86_XCPT_*).
1414	*/
1415	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1416	{
1417	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1418	{
1419	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1420	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1421	}
1422	}
1423
1424
1425	/**
1426	* Clears an exception intercept in the specified VMCB.
1427	*
1428	* @param pVCpu The cross context virtual CPU structure.
1429	* @param pVmcb Pointer to the VM control block.
1430	* @param uXcpt The exception (X86_XCPT_*).
1431	*
1432	* @remarks This takes into account if we're executing a nested-guest and only
1433	* removes the exception intercept if both the guest -and- nested-guest
1434	* are not intercepting it.
1435	*/
1436	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1437	{
1438	Assert(uXcpt != X86_XCPT_DB);
1439	Assert(uXcpt != X86_XCPT_AC);
1440	Assert(uXcpt != X86_XCPT_GP);
1441	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1442	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1443	{
1444	bool fRemove = true;
1445	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1446	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1447	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1448	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1449	{
1450	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1451	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1452	}
1453	# else
1454	RT_NOREF(pVCpu);
1455	# endif
1456	if (fRemove)
1457	{
1458	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1459	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1460	}
1461	}
1462	#else
1463	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1464	#endif
1465	}
1466
1467
1468	/**
1469	* Sets a control intercept in the specified VMCB.
1470	*
1471	* @param pVmcb Pointer to the VM control block.
1472	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1473	*/
1474	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1475	{
1476	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1477	{
1478	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1479	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1480	}
1481	}
1482
1483
1484	/**
1485	* Clears a control intercept in the specified VMCB.
1486	*
1487	* @returns @c true if the intercept is still set, @c false otherwise.
1488	* @param pVCpu The cross context virtual CPU structure.
1489	* @param pVmcb Pointer to the VM control block.
1490	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1491	*
1492	* @remarks This takes into account if we're executing a nested-guest and only
1493	* removes the control intercept if both the guest -and- nested-guest
1494	* are not intercepting it.
1495	*/
1496	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1497	{
1498	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1499	{
1500	bool fRemove = true;
1501	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1502	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1503	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1504	{
1505	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1506	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1507	}
1508	#else
1509	RT_NOREF(pVCpu);
1510	#endif
1511	if (fRemove)
1512	{
1513	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1514	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1515	}
1516	}
1517
1518	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1519	}
1520
1521
1522	/**
1523	* Exports the guest (or nested-guest) CR0 into the VMCB.
1524	*
1525	* @param pVCpu The cross context virtual CPU structure.
1526	* @param pVmcb Pointer to the VM control block.
1527	*
1528	* @remarks This assumes we always pre-load the guest FPU.
1529	* @remarks No-long-jump zone!!!
1530	*/
1531	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1532	{
1533	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1534
1535	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1536	uint64_t const uGuestCr0 = pCtx->cr0;
1537	uint64_t uShadowCr0 = uGuestCr0;
1538
1539	/* Always enable caching. */
1540	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1541
1542	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1543	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1544	{
1545	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1546	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1547	}
1548
1549	/*
1550	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1551	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1552	* see @bugref{7243#c103}.
1553	*/
1554	if (!(uGuestCr0 & X86_CR0_NE))
1555	{
1556	uShadowCr0 \|= X86_CR0_NE;
1557	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1558	}
1559	else
1560	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1561
1562	/*
1563	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1564	*
1565	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1566	* see @bugref{6944}.
1567	*
1568	* We also don't ever want to honor weird things like cache disable from the guest.
1569	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1570	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1571	*/
1572	if (uShadowCr0 == uGuestCr0)
1573	{
1574	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1575	{
1576	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1577	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1578	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1579	}
1580	else
1581	{
1582	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1583	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1584	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1585	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1586	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1587	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1588	}
1589	}
1590	else
1591	{
1592	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1593	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1594	}
1595	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1596
1597	Assert(!RT_HI_U32(uShadowCr0));
1598	if (pVmcb->guest.u64CR0 != uShadowCr0)
1599	{
1600	pVmcb->guest.u64CR0 = uShadowCr0;
1601	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1602	}
1603	}
1604
1605
1606	/**
1607	* Exports the guest (or nested-guest) CR3 into the VMCB.
1608	*
1609	* @param pVCpu The cross context virtual CPU structure.
1610	* @param pVmcb Pointer to the VM control block.
1611	*
1612	* @remarks No-long-jump zone!!!
1613	*/
1614	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1615	{
1616	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1617
1618	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1619	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1620	if (pVM->hmr0.s.fNestedPaging)
1621	{
1622	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1623	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1624	pVmcb->guest.u64CR3 = pCtx->cr3;
1625	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1626	}
1627	else
1628	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1629
1630	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1631	}
1632
1633
1634	/**
1635	* Exports the guest (or nested-guest) CR4 into the VMCB.
1636	*
1637	* @param pVCpu The cross context virtual CPU structure.
1638	* @param pVmcb Pointer to the VM control block.
1639	*
1640	* @remarks No-long-jump zone!!!
1641	*/
1642	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1643	{
1644	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1645
1646	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1647	uint64_t uShadowCr4 = pCtx->cr4;
1648	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1649	{
1650	switch (pVCpu->hm.s.enmShadowMode)
1651	{
1652	case PGMMODE_REAL:
1653	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1654	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1655
1656	case PGMMODE_32_BIT: /* 32-bit paging. */
1657	uShadowCr4 &= ~X86_CR4_PAE;
1658	break;
1659
1660	case PGMMODE_PAE: /* PAE paging. */
1661	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1662	/** Must use PAE paging as we could use physical memory > 4 GB */
1663	uShadowCr4 \|= X86_CR4_PAE;
1664	break;
1665
1666	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1667	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1668	#ifdef VBOX_WITH_64_BITS_GUESTS
1669	break;
1670	#else
1671	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1672	#endif
1673
1674	default: /* shut up gcc */
1675	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1676	}
1677	}
1678
1679	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1680	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1681	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1682	{
1683	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1684	hmR0SvmUpdateVmRunFunction(pVCpu);
1685	}
1686
1687	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1688	if (uShadowCr4 == pCtx->cr4)
1689	{
1690	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1691	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1692	else
1693	{
1694	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1695	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1696	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1697	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1698	}
1699	}
1700	else
1701	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1702
1703	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking PGM mode changes. */
1704	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1705
1706	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1707	Assert(!RT_HI_U32(uShadowCr4));
1708	pVmcb->guest.u64CR4 = uShadowCr4;
1709	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1710
1711	return VINF_SUCCESS;
1712	}
1713
1714
1715	/**
1716	* Exports the guest (or nested-guest) control registers into the VMCB.
1717	*
1718	* @returns VBox status code.
1719	* @param pVCpu The cross context virtual CPU structure.
1720	* @param pVmcb Pointer to the VM control block.
1721	*
1722	* @remarks No-long-jump zone!!!
1723	*/
1724	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1725	{
1726	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1727
1728	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1729	{
1730	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1731	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1732
1733	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1734	{
1735	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1736	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1737	}
1738
1739	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1740	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1741
1742	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1743	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1744	{
1745	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1746	if (RT_FAILURE(rc))
1747	return rc;
1748	}
1749
1750	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1751	}
1752	return VINF_SUCCESS;
1753	}
1754
1755
1756	/**
1757	* Exports the guest (or nested-guest) segment registers into the VMCB.
1758	*
1759	* @returns VBox status code.
1760	* @param pVCpu The cross context virtual CPU structure.
1761	* @param pVmcb Pointer to the VM control block.
1762	*
1763	* @remarks No-long-jump zone!!!
1764	*/
1765	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1766	{
1767	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1768	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1769
1770	/* Guest segment registers. */
1771	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1772	{
1773	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1774	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1775
1776	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1777	{
1778	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1779	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1780	}
1781
1782	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1783	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1784
1785	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1786	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1787
1788	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1789	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1790
1791	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1792	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1793
1794	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1795	}
1796
1797	/* Guest TR. */
1798	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1799	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1800
1801	/* Guest LDTR. */
1802	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1803	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1804
1805	/* Guest GDTR. */
1806	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1807	{
1808	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1809	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1810	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1811	}
1812
1813	/* Guest IDTR. */
1814	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1815	{
1816	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1817	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1818	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1819	}
1820
1821	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1822	\| HM_CHANGED_GUEST_TABLE_MASK);
1823	}
1824
1825
1826	/**
1827	* Exports the guest (or nested-guest) MSRs into the VMCB.
1828	*
1829	* @param pVCpu The cross context virtual CPU structure.
1830	* @param pVmcb Pointer to the VM control block.
1831	*
1832	* @remarks No-long-jump zone!!!
1833	*/
1834	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1835	{
1836	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1837	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1838
1839	/* Guest Sysenter MSRs. */
1840	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1841	{
1842	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1843	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1844
1845	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1846	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1847
1848	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1849	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1850	}
1851
1852	/*
1853	* Guest EFER MSR.
1854	* AMD-V requires guest EFER.SVME to be set. Weird.
1855	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1856	*/
1857	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1858	{
1859	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1860	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1861	}
1862
1863	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1864	if ( !CPUMIsGuestInLongModeEx(pCtx)
1865	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1866	{
1867	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1868	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1869	}
1870
1871	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1872	{
1873	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1874	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1875	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1876	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1877	}
1878
1879	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1880	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1881
1882	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1883	\| HM_CHANGED_GUEST_EFER_MSR
1884	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1885	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1886
1887	/*
1888	* Setup the PAT MSR (applicable for Nested Paging only).
1889	*
1890	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1891	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1892	*
1893	* While guests can modify and see the modified values through the shadow values,
1894	* we shall not honor any guest modifications of this MSR to ensure caching is always
1895	* enabled similar to how we clear CR0.CD and NW bits.
1896	*
1897	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1898	*/
1899	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1900
1901	/* Enable the last branch record bit if LBR virtualization is enabled. */
1902	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1903	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1904	}
1905
1906
1907	/**
1908	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1909	* the necessary intercepts accordingly.
1910	*
1911	* @param pVCpu The cross context virtual CPU structure.
1912	* @param pVmcb Pointer to the VM control block.
1913	*
1914	* @remarks No-long-jump zone!!!
1915	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1916	*/
1917	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1918	{
1919	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1920
1921	/*
1922	* Anyone single stepping on the host side? If so, we'll have to use the
1923	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1924	* the VMM level like the VT-x implementations does.
1925	*/
1926	bool fInterceptMovDRx = false;
1927	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1928	if (fStepping)
1929	{
1930	pVCpu->hmr0.s.fClearTrapFlag = true;
1931	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1932	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1933	}
1934
1935	if ( fStepping
1936	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1937	{
1938	/*
1939	* Use the combined guest and host DRx values found in the hypervisor
1940	* register set because the debugger has breakpoints active or someone
1941	* is single stepping on the host side.
1942	*
1943	* Note! DBGF expects a clean DR6 state before executing guest code.
1944	*/
1945	if (!CPUMIsHyperDebugStateActive(pVCpu))
1946	{
1947	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1948	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1949	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1950	}
1951
1952	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1953	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1954	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1955	{
1956	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1957	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1958	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1959	}
1960
1961	/** @todo If we cared, we could optimize to allow the guest to read registers
1962	* with the same values. */
1963	fInterceptMovDRx = true;
1964	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1965	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1966	}
1967	else
1968	{
1969	/*
1970	* Update DR6, DR7 with the guest values if necessary.
1971	*/
1972	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1973	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1974	{
1975	pVmcb->guest.u64DR7 = pCtx->dr[7];
1976	pVmcb->guest.u64DR6 = pCtx->dr[6];
1977	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1978	}
1979	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1980
1981	/*
1982	* If the guest has enabled debug registers, we need to load them prior to
1983	* executing guest code so they'll trigger at the right time.
1984	*/
1985	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1986	{
1987	if (!CPUMIsGuestDebugStateActive(pVCpu))
1988	{
1989	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1990	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1991	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1992	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1993	}
1994	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
1995	}
1996	/*
1997	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
1998	* intercept #DB as DR6 is updated in the VMCB.
1999	*
2000	* Note! If we cared and dared, we could skip intercepting \#DB here.
2001	* However, \#DB shouldn't be performance critical, so we'll play safe
2002	* and keep the code similar to the VT-x code and always intercept it.
2003	*/
2004	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2005	fInterceptMovDRx = true;
2006	}
2007
2008	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2009	if (fInterceptMovDRx)
2010	{
2011	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2012	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2013	{
2014	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2015	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2016	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2017	}
2018	}
2019	else
2020	{
2021	if ( pVmcb->ctrl.u16InterceptRdDRx
2022	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2023	{
2024	pVmcb->ctrl.u16InterceptRdDRx = 0;
2025	pVmcb->ctrl.u16InterceptWrDRx = 0;
2026	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2027	}
2028	}
2029	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2030	}
2031
2032	/**
2033	* Exports the hardware virtualization state into the nested-guest
2034	* VMCB.
2035	*
2036	* @param pVCpu The cross context virtual CPU structure.
2037	* @param pVmcb Pointer to the VM control block.
2038	*
2039	* @remarks No-long-jump zone!!!
2040	*/
2041	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2042	{
2043	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2044
2045	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2046	{
2047	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2048	{
2049	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2050	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2051
2052	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2053	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2054	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2055	NOREF(pVM);
2056
2057	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2058	}
2059
2060	/*
2061	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2062	* since SVM doesn't have a preemption timer.
2063	*
2064	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2065	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2066	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2067	*/
2068	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2069	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2070	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2071	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2072	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2073	{
2074	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2075	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2076	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2077	}
2078	else
2079	{
2080	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2081	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2082	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2083	}
2084	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2085
2086	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2087	}
2088	}
2089
2090
2091	/**
2092	* Exports the guest APIC TPR state into the VMCB.
2093	*
2094	* @returns VBox status code.
2095	* @param pVCpu The cross context virtual CPU structure.
2096	* @param pVmcb Pointer to the VM control block.
2097	*/
2098	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2099	{
2100	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2101
2102	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2103	{
2104	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2105	if ( PDMHasApic(pVM)
2106	&& APICIsEnabled(pVCpu))
2107	{
2108	bool fPendingIntr;
2109	uint8_t u8Tpr;
2110	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2111	AssertRCReturn(rc, rc);
2112
2113	/* Assume that we need to trap all TPR accesses and thus need not check on
2114	every #VMEXIT if we should update the TPR. */
2115	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2116	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2117
2118	if (!pVM->hm.s.fTprPatchingActive)
2119	{
2120	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2121	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2122
2123	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2124	can deliver the interrupt to the guest. */
2125	if (fPendingIntr)
2126	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2127	else
2128	{
2129	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2130	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2131	}
2132
2133	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2134	}
2135	else
2136	{
2137	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2138	pVmcb->guest.u64LSTAR = u8Tpr;
2139	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2140
2141	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2142	if (fPendingIntr)
2143	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2144	else
2145	{
2146	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2147	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2148	}
2149	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2150	}
2151	}
2152	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2153	}
2154	return VINF_SUCCESS;
2155	}
2156
2157
2158	/**
2159	* Sets up the exception interrupts required for guest execution in the VMCB.
2160	*
2161	* @param pVCpu The cross context virtual CPU structure.
2162	* @param pVmcb Pointer to the VM control block.
2163	*
2164	* @remarks No-long-jump zone!!!
2165	*/
2166	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2167	{
2168	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2169
2170	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2171	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2172	{
2173	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2174	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2175	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2176	else
2177	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2178
2179	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2180	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2181	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2182	else
2183	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2184
2185	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2186	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2187	}
2188	}
2189
2190
2191	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2192	/**
2193	* Merges guest and nested-guest intercepts for executing the nested-guest using
2194	* hardware-assisted SVM.
2195	*
2196	* This merges the guest and nested-guest intercepts in a way that if the outer
2197	* guest intercept is set we need to intercept it in the nested-guest as
2198	* well.
2199	*
2200	* @param pVCpu The cross context virtual CPU structure.
2201	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2202	*/
2203	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2204	{
2205	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2206	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2207	PSVMVMCB pVmcbNstGst = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
2208	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2209
2210	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2211	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2212	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2213
2214	/* Always intercept CR4 writes for tracking PGM mode changes. */
2215	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2216
2217	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2218	if (!pVM->hmr0.s.fNestedPaging)
2219	{
2220	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2221	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2222	}
2223
2224	/** @todo Figure out debugging with nested-guests, till then just intercept
2225	* all DR[0-15] accesses. */
2226	pVmcbNstGstCtrl->u16InterceptRdDRx \|= 0xffff;
2227	pVmcbNstGstCtrl->u16InterceptWrDRx \|= 0xffff;
2228
2229	/*
2230	* Merge the guest's exception intercepts into the nested-guest VMCB.
2231	*
2232	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2233	* while executing the nested-guest.
2234	*
2235	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2236	* be tweaked later depending on how we wish to implement breakpoints.
2237	*
2238	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2239	* loaded into their guests, not ours.
2240	*
2241	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2242	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2243	*/
2244	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2245	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2246	& ~( RT_BIT(X86_XCPT_UD)
2247	\| RT_BIT(X86_XCPT_BP)
2248	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2249	#else
2250	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2251	#endif
2252
2253	/*
2254	* Adjust intercepts while executing the nested-guest that differ from the
2255	* outer guest intercepts.
2256	*
2257	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2258	* that belong to the nested-guest to the outer guest.
2259	*
2260	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2261	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2262	*/
2263	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2264	\| SVM_CTRL_INTERCEPT_VMMCALL))
2265	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2266
2267	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2268	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2269
2270	/* Finally, update the VMCB clean bits. */
2271	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2272	}
2273	#endif
2274
2275
2276	/**
2277	* Enters the AMD-V session.
2278	*
2279	* @returns VBox status code.
2280	* @param pVCpu The cross context virtual CPU structure.
2281	*/
2282	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2283	{
2284	AssertPtr(pVCpu);
2285	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2286	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2287
2288	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2289	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2290	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2291
2292	pVCpu->hmr0.s.fLeaveDone = false;
2293	return VINF_SUCCESS;
2294	}
2295
2296
2297	/**
2298	* Thread-context callback for AMD-V.
2299	*
2300	* @param enmEvent The thread-context event.
2301	* @param pVCpu The cross context virtual CPU structure.
2302	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2303	* @thread EMT(pVCpu)
2304	*/
2305	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2306	{
2307	NOREF(fGlobalInit);
2308
2309	switch (enmEvent)
2310	{
2311	case RTTHREADCTXEVENT_OUT:
2312	{
2313	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2314	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2315	VMCPU_ASSERT_EMT(pVCpu);
2316
2317	/* No longjmps (log-flush, locks) in this fragile context. */
2318	VMMRZCallRing3Disable(pVCpu);
2319
2320	if (!pVCpu->hmr0.s.fLeaveDone)
2321	{
2322	hmR0SvmLeave(pVCpu, false /* fImportState */);
2323	pVCpu->hmr0.s.fLeaveDone = true;
2324	}
2325
2326	/* Leave HM context, takes care of local init (term). */
2327	int rc = HMR0LeaveCpu(pVCpu);
2328	AssertRC(rc); NOREF(rc);
2329
2330	/* Restore longjmp state. */
2331	VMMRZCallRing3Enable(pVCpu);
2332	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2333	break;
2334	}
2335
2336	case RTTHREADCTXEVENT_IN:
2337	{
2338	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2339	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2340	VMCPU_ASSERT_EMT(pVCpu);
2341
2342	/* No longjmps (log-flush, locks) in this fragile context. */
2343	VMMRZCallRing3Disable(pVCpu);
2344
2345	/*
2346	* Initialize the bare minimum state required for HM. This takes care of
2347	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2348	*/
2349	int rc = hmR0EnterCpu(pVCpu);
2350	AssertRC(rc); NOREF(rc);
2351	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2352	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2353
2354	pVCpu->hmr0.s.fLeaveDone = false;
2355
2356	/* Restore longjmp state. */
2357	VMMRZCallRing3Enable(pVCpu);
2358	break;
2359	}
2360
2361	default:
2362	break;
2363	}
2364	}
2365
2366
2367	/**
2368	* Saves the host state.
2369	*
2370	* @returns VBox status code.
2371	* @param pVCpu The cross context virtual CPU structure.
2372	*
2373	* @remarks No-long-jump zone!!!
2374	*/
2375	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2376	{
2377	NOREF(pVCpu);
2378
2379	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2380	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2381	return VINF_SUCCESS;
2382	}
2383
2384
2385	/**
2386	* Exports the guest or nested-guest state from the virtual-CPU context into the
2387	* VMCB.
2388	*
2389	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2390	* code based on the virtual-CPU mode.
2391	*
2392	* @returns VBox status code.
2393	* @param pVCpu The cross context virtual CPU structure.
2394	* @param pSvmTransient Pointer to the SVM-transient structure.
2395	*
2396	* @remarks No-long-jump zone!!!
2397	*/
2398	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2399	{
2400	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2401
2402	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2403	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2404	Assert(pVmcb);
2405
2406	pVmcb->guest.u64RIP = pCtx->rip;
2407	pVmcb->guest.u64RSP = pCtx->rsp;
2408	pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2409	pVmcb->guest.u64RAX = pCtx->rax;
2410
2411	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2412	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2413
2414	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2415	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2416	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2417	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2418	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2419
2420	ASMSetFlags(fEFlags);
2421
2422	if (!fIsNestedGuest)
2423	{
2424	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2425	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2426	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2427	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2428	}
2429
2430	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2431	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2432	\| HM_CHANGED_GUEST_RFLAGS
2433	\| HM_CHANGED_GUEST_GPRS_MASK
2434	\| HM_CHANGED_GUEST_X87
2435	\| HM_CHANGED_GUEST_SSE_AVX
2436	\| HM_CHANGED_GUEST_OTHER_XSAVE
2437	\| HM_CHANGED_GUEST_XCRx
2438	\| HM_CHANGED_GUEST_TSC_AUX
2439	\| HM_CHANGED_GUEST_OTHER_MSRS;
2440	if (fIsNestedGuest)
2441	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2442	\| HM_CHANGED_GUEST_APIC_TPR;
2443
2444	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2445	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2446
2447	#ifdef VBOX_STRICT
2448	/*
2449	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2450	* except for the host-context and/or shared host-guest context bits.
2451	*/
2452	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2453	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2454	("fCtxChanged=%#RX64\n", fCtxChanged));
2455
2456	/*
2457	* If we need to log state that isn't always imported, we'll need to import them here.
2458	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2459	*/
2460	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2461	#endif
2462
2463	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2464	return VINF_SUCCESS;
2465	}
2466
2467
2468	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2469	/**
2470	* Merges the guest and nested-guest MSR permission bitmap.
2471	*
2472	* If the guest is intercepting an MSR we need to intercept it regardless of
2473	* whether the nested-guest is intercepting it or not.
2474	*
2475	* @param pHostCpu The HM physical-CPU structure.
2476	* @param pVCpu The cross context virtual CPU structure.
2477	*
2478	* @remarks No-long-jmp zone!!!
2479	*/
2480	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2481	{
2482	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2483	uint64_t const pu64NstGstMsrpm = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
2484	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2485
2486	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2487	uint32_t const offRsvdQwords = 0x1800 >> 3;
2488	for (uint32_t i = 0; i < offRsvdQwords; i++)
2489	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2490	}
2491
2492
2493	/**
2494	* Caches the nested-guest VMCB fields before we modify them for execution using
2495	* hardware-assisted SVM.
2496	*
2497	* @returns true if the VMCB was previously already cached, false otherwise.
2498	* @param pVCpu The cross context virtual CPU structure.
2499	*
2500	* @sa HMNotifySvmNstGstVmexit.
2501	*/
2502	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2503	{
2504	/*
2505	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2506	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2507	*
2508	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2509	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2510	*/
2511	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2512	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2513	if (!fWasCached)
2514	{
2515	PCSVMVMCB pVmcbNstGst = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
2516	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2517	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2518	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2519	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2520	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2521	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2522	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2523	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2524	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2525	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2526	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2527	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2528	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2529	pVmcbNstGstCache->fCacheValid = true;
2530	Log4Func(("Cached VMCB fields\n"));
2531	}
2532
2533	return fWasCached;
2534	}
2535
2536
2537	/**
2538	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2539	*
2540	* This is done the first time we enter nested-guest execution using SVM R0
2541	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2542	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2543	*
2544	* @param pVCpu The cross context virtual CPU structure.
2545	*/
2546	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2547	{
2548	PSVMVMCB pVmcbNstGst = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
2549	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2550
2551	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2552
2553	/*
2554	* First cache the nested-guest VMCB fields we may potentially modify.
2555	*/
2556	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2557	if (!fVmcbCached)
2558	{
2559	/*
2560	* The IOPM of the nested-guest can be ignored because the the guest always
2561	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2562	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2563	*/
2564	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2565
2566	/*
2567	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2568	* nested-paging suddenly while executing a VM (see assertion at the end of
2569	* Trap0eHandler() in PGMAllBth.h).
2570	*/
2571	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2572
2573	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2574	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2575
2576	/*
2577	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2578	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2579	*/
2580	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2581
2582	#ifdef DEBUG_ramshankar
2583	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2584	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2585	#endif
2586
2587	/*
2588	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2589	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2590	*/
2591	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2592	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2593	\| SVM_CTRL_INTERCEPT_VMLOAD;
2594
2595	/*
2596	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2597	* CLGI/STGI instructions executed by the nested-guest.
2598	*/
2599	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2600	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2601	\| SVM_CTRL_INTERCEPT_STGI;
2602
2603	/* Merge the guest and nested-guest intercepts. */
2604	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2605
2606	/* Update the VMCB clean bits. */
2607	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2608	}
2609	else
2610	{
2611	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2612	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2613	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2614	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2615	}
2616	}
2617	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2618
2619
2620	/**
2621	* Exports the state shared between the host and guest (or nested-guest) into
2622	* the VMCB.
2623	*
2624	* @param pVCpu The cross context virtual CPU structure.
2625	* @param pVmcb Pointer to the VM control block.
2626	*
2627	* @remarks No-long-jump zone!!!
2628	*/
2629	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2630	{
2631	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2632	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2633
2634	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2635	{
2636	/** @todo Figure out stepping with nested-guest. */
2637	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2638	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
2639	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2640	else
2641	{
2642	pVmcb->guest.u64DR6 = pCtx->dr[6];
2643	pVmcb->guest.u64DR7 = pCtx->dr[7];
2644	}
2645	}
2646
2647	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2648	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2649	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2650	}
2651
2652
2653	/**
2654	* Worker for SVMR0ImportStateOnDemand.
2655	*
2656	* @param pVCpu The cross context virtual CPU structure.
2657	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2658	*/
2659	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2660	{
2661	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2662
2663	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2664	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2665	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2666	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2667
2668	/*
2669	* We disable interrupts to make the updating of the state and in particular
2670	* the fExtrn modification atomic wrt to preemption hooks.
2671	*/
2672	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2673
2674	fWhat &= pCtx->fExtrn;
2675	if (fWhat)
2676	{
2677	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2678	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2679	{
2680	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2681	{
2682	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2683	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2684	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2685	}
2686	}
2687
2688	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2689	{
2690	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2691	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2692	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2693	}
2694	#endif
2695
2696	if (fWhat & CPUMCTX_EXTRN_HM_SVM_INT_SHADOW)
2697	{
2698	if (pVmcbCtrl->IntShadow.n.u1IntShadow)
2699	EMSetInhibitInterruptsPC(pVCpu, pVmcbGuest->u64RIP);
2700	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2701	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2702	}
2703
2704	if (fWhat & CPUMCTX_EXTRN_RIP)
2705	pCtx->rip = pVmcbGuest->u64RIP;
2706
2707	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2708	pCtx->eflags.u32 = pVmcbGuest->u64RFlags;
2709
2710	if (fWhat & CPUMCTX_EXTRN_RSP)
2711	pCtx->rsp = pVmcbGuest->u64RSP;
2712
2713	if (fWhat & CPUMCTX_EXTRN_RAX)
2714	pCtx->rax = pVmcbGuest->u64RAX;
2715
2716	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2717	{
2718	if (fWhat & CPUMCTX_EXTRN_CS)
2719	{
2720	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2721	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2722	/** @todo SELM might need to be fixed as it too should not care about the
2723	* granularity bit. See @bugref{6785}. */
2724	if ( !pCtx->cs.Attr.n.u1Granularity
2725	&& pCtx->cs.Attr.n.u1Present
2726	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2727	{
2728	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2729	pCtx->cs.Attr.n.u1Granularity = 1;
2730	}
2731	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2732	}
2733	if (fWhat & CPUMCTX_EXTRN_SS)
2734	{
2735	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2736	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2737	/*
2738	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2739	* VMCB and uses that and thus it's possible that when the CPL changes during
2740	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2741	* AMD Fusion CPUs with 64-bit guests.
2742	*
2743	* See AMD spec. 15.5.1 "Basic operation".
2744	*/
2745	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2746	uint8_t const uCpl = pVmcbGuest->u8CPL;
2747	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2748	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2749	}
2750	if (fWhat & CPUMCTX_EXTRN_DS)
2751	{
2752	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2753	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2754	}
2755	if (fWhat & CPUMCTX_EXTRN_ES)
2756	{
2757	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2758	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2759	}
2760	if (fWhat & CPUMCTX_EXTRN_FS)
2761	{
2762	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2763	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2764	}
2765	if (fWhat & CPUMCTX_EXTRN_GS)
2766	{
2767	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2768	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2769	}
2770	}
2771
2772	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2773	{
2774	if (fWhat & CPUMCTX_EXTRN_TR)
2775	{
2776	/*
2777	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2778	* are used between Intel and AMD, see @bugref{6208#c39}.
2779	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2780	*/
2781	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2782	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2783	{
2784	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2785	\|\| CPUMIsGuestInLongModeEx(pCtx))
2786	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2787	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2788	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2789	}
2790	}
2791
2792	if (fWhat & CPUMCTX_EXTRN_LDTR)
2793	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2794
2795	if (fWhat & CPUMCTX_EXTRN_GDTR)
2796	{
2797	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2798	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2799	}
2800
2801	if (fWhat & CPUMCTX_EXTRN_IDTR)
2802	{
2803	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2804	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2805	}
2806	}
2807
2808	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2809	{
2810	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2811	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2812	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2813	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2814	}
2815
2816	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2817	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2818	{
2819	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2820	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2821	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2822	}
2823
2824	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2825	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2826
2827	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2828	{
2829	if (fWhat & CPUMCTX_EXTRN_DR6)
2830	{
2831	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2832	pCtx->dr[6] = pVmcbGuest->u64DR6;
2833	else
2834	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2835	}
2836
2837	if (fWhat & CPUMCTX_EXTRN_DR7)
2838	{
2839	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2840	pCtx->dr[7] = pVmcbGuest->u64DR7;
2841	else
2842	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2843	}
2844	}
2845
2846	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2847	{
2848	if (fWhat & CPUMCTX_EXTRN_CR0)
2849	{
2850	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2851	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2852	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2853	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2854	CPUMSetGuestCR0(pVCpu, uCr0);
2855	VMMRZCallRing3Enable(pVCpu);
2856	}
2857
2858	if (fWhat & CPUMCTX_EXTRN_CR2)
2859	pCtx->cr2 = pVmcbGuest->u64CR2;
2860
2861	if (fWhat & CPUMCTX_EXTRN_CR3)
2862	{
2863	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2864	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2865	{
2866	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2867	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2868	}
2869	}
2870
2871	/* Changes to CR4 are always intercepted. */
2872	}
2873
2874	/* Update fExtrn. */
2875	pCtx->fExtrn &= ~fWhat;
2876
2877	/* If everything has been imported, clear the HM keeper bit. */
2878	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2879	{
2880	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2881	Assert(!pCtx->fExtrn);
2882	}
2883	}
2884	else
2885	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2886
2887	ASMSetFlags(fEFlags);
2888
2889	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2890
2891	/*
2892	* Honor any pending CR3 updates.
2893	*
2894	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2895	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2896	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2897	* handling -> hmR0SvmImportGuestState() and here we are.
2898	*
2899	* The reason for such complicated handling is because VM-exits that call into PGM expect
2900	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2901	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2902	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2903	*
2904	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2905	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2906	*/
2907	if ( VMMRZCallRing3IsEnabled(pVCpu)
2908	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2909	{
2910	Assert(pCtx->cr3 == pVmcbGuest->u64CR3);
2911	PGMUpdateCR3(pVCpu, pCtx->cr3);
2912	}
2913	}
2914
2915
2916	/**
2917	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2918	* context.
2919	*
2920	* Currently there is no residual state left in the CPU that is not updated in the
2921	* VMCB.
2922	*
2923	* @returns VBox status code.
2924	* @param pVCpu The cross context virtual CPU structure.
2925	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2926	*/
2927	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2928	{
2929	hmR0SvmImportGuestState(pVCpu, fWhat);
2930	return VINF_SUCCESS;
2931	}
2932
2933
2934	/**
2935	* Does the necessary state syncing before returning to ring-3 for any reason
2936	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2937	*
2938	* @param pVCpu The cross context virtual CPU structure.
2939	* @param fImportState Whether to import the guest state from the VMCB back
2940	* to the guest-CPU context.
2941	*
2942	* @remarks No-long-jmp zone!!!
2943	*/
2944	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2945	{
2946	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2947	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2948	Assert(VMMR0IsLogFlushDisabled(pVCpu));
2949
2950	/*
2951	* !!! IMPORTANT !!!
2952	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2953	*/
2954
2955	/* Save the guest state if necessary. */
2956	if (fImportState)
2957	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2958
2959	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2960	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2961	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2962
2963	/*
2964	* Restore host debug registers if necessary and resync on next R0 reentry.
2965	*/
2966	#ifdef VBOX_STRICT
2967	if (CPUMIsHyperDebugStateActive(pVCpu))
2968	{
2969	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2970	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2971	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2972	}
2973	#endif
2974	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2975	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2976	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2977
2978	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
2979	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
2980	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
2981	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
2982	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
2983	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
2984	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2985
2986	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
2987	}
2988
2989
2990	/**
2991	* Leaves the AMD-V session.
2992	*
2993	* Only used while returning to ring-3 either due to longjump or exits to
2994	* ring-3.
2995	*
2996	* @returns VBox status code.
2997	* @param pVCpu The cross context virtual CPU structure.
2998	*/
2999	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3000	{
3001	HM_DISABLE_PREEMPT(pVCpu);
3002	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3003	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3004
3005	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3006	and done this from the SVMR0ThreadCtxCallback(). */
3007	if (!pVCpu->hmr0.s.fLeaveDone)
3008	{
3009	hmR0SvmLeave(pVCpu, true /* fImportState */);
3010	pVCpu->hmr0.s.fLeaveDone = true;
3011	}
3012
3013	/*
3014	* !!! IMPORTANT !!!
3015	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3016	*/
3017
3018	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3019	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3020	VMMR0ThreadCtxHookDisable(pVCpu);
3021
3022	/* Leave HM context. This takes care of local init (term). */
3023	int rc = HMR0LeaveCpu(pVCpu);
3024
3025	HM_RESTORE_PREEMPT();
3026	return rc;
3027	}
3028
3029
3030	/**
3031	* Does the necessary state syncing before doing a longjmp to ring-3.
3032	*
3033	* @returns VBox status code.
3034	* @param pVCpu The cross context virtual CPU structure.
3035	*
3036	* @remarks No-long-jmp zone!!!
3037	*/
3038	static int hmR0SvmLongJmpToRing3(PVMCPUCC pVCpu)
3039	{
3040	return hmR0SvmLeaveSession(pVCpu);
3041	}
3042
3043
3044	/**
3045	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3046	* any remaining host state) before we longjump to ring-3 and possibly get
3047	* preempted.
3048	*
3049	* @param pVCpu The cross context virtual CPU structure.
3050	* @param enmOperation The operation causing the ring-3 longjump.
3051	*/
3052	VMMR0DECL(int) SVMR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
3053	{
3054	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
3055	{
3056	/*
3057	* !!! IMPORTANT !!!
3058	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3059	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3060	*/
3061	VMMRZCallRing3RemoveNotification(pVCpu);
3062	VMMRZCallRing3Disable(pVCpu);
3063	HM_DISABLE_PREEMPT(pVCpu);
3064
3065	/* Import the entire guest state. */
3066	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3067
3068	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3069	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3070
3071	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3072	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3073
3074	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3075	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3076	VMMR0ThreadCtxHookDisable(pVCpu);
3077
3078	/* Leave HM context. This takes care of local init (term). */
3079	HMR0LeaveCpu(pVCpu);
3080
3081	HM_RESTORE_PREEMPT();
3082	return VINF_SUCCESS;
3083	}
3084
3085	Assert(pVCpu);
3086	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3087	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3088
3089	VMMRZCallRing3Disable(pVCpu);
3090	Assert(VMMR0IsLogFlushDisabled(pVCpu));
3091
3092	Log4Func(("Calling hmR0SvmLongJmpToRing3\n"));
3093	int rc = hmR0SvmLongJmpToRing3(pVCpu);
3094	AssertRCReturn(rc, rc);
3095
3096	VMMRZCallRing3Enable(pVCpu);
3097	return VINF_SUCCESS;
3098	}
3099
3100
3101	/**
3102	* Take necessary actions before going back to ring-3.
3103	*
3104	* An action requires us to go back to ring-3. This function does the necessary
3105	* steps before we can safely return to ring-3. This is not the same as longjmps
3106	* to ring-3, this is voluntary.
3107	*
3108	* @returns Strict VBox status code.
3109	* @param pVCpu The cross context virtual CPU structure.
3110	* @param rcExit The reason for exiting to ring-3. Can be
3111	* VINF_VMM_UNKNOWN_RING3_CALL.
3112	*/
3113	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3114	{
3115	Assert(pVCpu);
3116	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3117
3118	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3119	VMMRZCallRing3Disable(pVCpu);
3120	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3121	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3122
3123	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3124	if (pVCpu->hm.s.Event.fPending)
3125	{
3126	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3127	Assert(!pVCpu->hm.s.Event.fPending);
3128	}
3129
3130	/* Sync. the necessary state for going back to ring-3. */
3131	hmR0SvmLeaveSession(pVCpu);
3132	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3133
3134	/* Thread-context hooks are unregistered at this point!!! */
3135	/* Ring-3 callback notifications are unregistered at this point!!! */
3136
3137	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3138	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3139	\| CPUM_CHANGED_LDTR
3140	\| CPUM_CHANGED_GDTR
3141	\| CPUM_CHANGED_IDTR
3142	\| CPUM_CHANGED_TR
3143	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3144	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3145	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3146	{
3147	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3148	}
3149
3150	/* Update the exit-to-ring 3 reason. */
3151	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3152
3153	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3154	if ( rcExit != VINF_EM_RAW_INTERRUPT
3155	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3156	{
3157	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3158	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3159	}
3160
3161	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3162	VMMRZCallRing3Enable(pVCpu);
3163
3164	/*
3165	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3166	* and if we're injecting an event we should have a TRPM trap pending.
3167	*/
3168	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3169	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3170	VERR_SVM_IPE_5);
3171	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3172	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3173	VERR_SVM_IPE_4);
3174
3175	return rcExit;
3176	}
3177
3178
3179	/**
3180	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3181	* intercepts.
3182	*
3183	* @param pVCpu The cross context virtual CPU structure.
3184	* @param pVmcb Pointer to the VM control block.
3185	*
3186	* @remarks No-long-jump zone!!!
3187	*/
3188	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3189	{
3190	/*
3191	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3192	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3193	* RDTSC/RDTSCP as well.
3194	*/
3195	bool fParavirtTsc;
3196	uint64_t uTscOffset;
3197	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3198
3199	bool fIntercept;
3200	if (fCanUseRealTsc)
3201	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3202	else
3203	{
3204	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3205	fIntercept = true;
3206	}
3207
3208	if (!fIntercept)
3209	{
3210	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3211	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3212	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3213	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3214	#endif
3215
3216	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3217	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3218	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3219	}
3220
3221	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3222	information before every VM-entry, hence we have nothing to do here at the moment. */
3223	if (fParavirtTsc)
3224	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3225	}
3226
3227
3228	/**
3229	* Sets an event as a pending event to be injected into the guest.
3230	*
3231	* @param pVCpu The cross context virtual CPU structure.
3232	* @param pEvent Pointer to the SVM event.
3233	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3234	* page-fault.
3235	*
3236	* @remarks Statistics counter assumes this is a guest event being reflected to
3237	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3238	*/
3239	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3240	{
3241	Assert(!pVCpu->hm.s.Event.fPending);
3242	Assert(pEvent->n.u1Valid);
3243
3244	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3245	pVCpu->hm.s.Event.fPending = true;
3246	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3247
3248	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3249	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3250	}
3251
3252
3253	/**
3254	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3255	*
3256	* @param pVCpu The cross context virtual CPU structure.
3257	*/
3258	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3259	{
3260	SVMEVENT Event;
3261	Event.u = 0;
3262	Event.n.u1Valid = 1;
3263	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3264	Event.n.u8Vector = X86_XCPT_UD;
3265	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3266	}
3267
3268
3269	/**
3270	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3271	*
3272	* @param pVCpu The cross context virtual CPU structure.
3273	*/
3274	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3275	{
3276	SVMEVENT Event;
3277	Event.u = 0;
3278	Event.n.u1Valid = 1;
3279	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3280	Event.n.u8Vector = X86_XCPT_DB;
3281	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3282	}
3283
3284
3285	/**
3286	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3287	*
3288	* @param pVCpu The cross context virtual CPU structure.
3289	* @param u32ErrCode The error-code for the page-fault.
3290	* @param uFaultAddress The page fault address (CR2).
3291	*
3292	* @remarks This updates the guest CR2 with @a uFaultAddress!
3293	*/
3294	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3295	{
3296	SVMEVENT Event;
3297	Event.u = 0;
3298	Event.n.u1Valid = 1;
3299	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3300	Event.n.u8Vector = X86_XCPT_PF;
3301	Event.n.u1ErrorCodeValid = 1;
3302	Event.n.u32ErrorCode = u32ErrCode;
3303
3304	/* Update CR2 of the guest. */
3305	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3306	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3307	{
3308	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3309	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3310	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3311	}
3312
3313	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3314	}
3315
3316
3317	/**
3318	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3319	*
3320	* @param pVCpu The cross context virtual CPU structure.
3321	*/
3322	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3323	{
3324	SVMEVENT Event;
3325	Event.u = 0;
3326	Event.n.u1Valid = 1;
3327	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3328	Event.n.u8Vector = X86_XCPT_MF;
3329	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3330	}
3331
3332
3333	/**
3334	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3335	*
3336	* @param pVCpu The cross context virtual CPU structure.
3337	*/
3338	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3339	{
3340	SVMEVENT Event;
3341	Event.u = 0;
3342	Event.n.u1Valid = 1;
3343	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3344	Event.n.u8Vector = X86_XCPT_DF;
3345	Event.n.u1ErrorCodeValid = 1;
3346	Event.n.u32ErrorCode = 0;
3347	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3348	}
3349
3350
3351	/**
3352	* Injects an event into the guest upon VMRUN by updating the relevant field
3353	* in the VMCB.
3354	*
3355	* @param pVCpu The cross context virtual CPU structure.
3356	* @param pVmcb Pointer to the guest VM control block.
3357	* @param pEvent Pointer to the event.
3358	*
3359	* @remarks No-long-jump zone!!!
3360	* @remarks Requires CR0!
3361	*/
3362	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3363	{
3364	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3365	pVmcb->ctrl.EventInject.u = pEvent->u;
3366	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3367	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3368	{
3369	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3370	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedXcptsR0[pEvent->n.u8Vector]);
3371	}
3372	else
3373	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3374	RT_NOREF(pVCpu);
3375
3376	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3377	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3378	}
3379
3380
3381
3382	/**
3383	* Converts any TRPM trap into a pending HM event. This is typically used when
3384	* entering from ring-3 (not longjmp returns).
3385	*
3386	* @param pVCpu The cross context virtual CPU structure.
3387	*/
3388	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3389	{
3390	Assert(TRPMHasTrap(pVCpu));
3391	Assert(!pVCpu->hm.s.Event.fPending);
3392
3393	uint8_t uVector;
3394	TRPMEVENT enmTrpmEvent;
3395	uint32_t uErrCode;
3396	RTGCUINTPTR GCPtrFaultAddress;
3397	uint8_t cbInstr;
3398
3399	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3400	AssertRC(rc);
3401
3402	SVMEVENT Event;
3403	Event.u = 0;
3404	Event.n.u1Valid = 1;
3405	Event.n.u8Vector = uVector;
3406
3407	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3408	if (enmTrpmEvent == TRPM_TRAP)
3409	{
3410	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3411	switch (uVector)
3412	{
3413	case X86_XCPT_NMI:
3414	{
3415	Event.n.u3Type = SVM_EVENT_NMI;
3416	break;
3417	}
3418
3419	case X86_XCPT_BP:
3420	case X86_XCPT_OF:
3421	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3422	RT_FALL_THRU();
3423
3424	case X86_XCPT_PF:
3425	case X86_XCPT_DF:
3426	case X86_XCPT_TS:
3427	case X86_XCPT_NP:
3428	case X86_XCPT_SS:
3429	case X86_XCPT_GP:
3430	case X86_XCPT_AC:
3431	{
3432	Event.n.u1ErrorCodeValid = 1;
3433	Event.n.u32ErrorCode = uErrCode;
3434	break;
3435	}
3436	}
3437	}
3438	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3439	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3440	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3441	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3442	else
3443	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3444
3445	rc = TRPMResetTrap(pVCpu);
3446	AssertRC(rc);
3447
3448	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3449	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3450
3451	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3452	}
3453
3454
3455	/**
3456	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3457	* AMD-V to execute any instruction.
3458	*
3459	* @param pVCpu The cross context virtual CPU structure.
3460	*/
3461	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3462	{
3463	Assert(pVCpu->hm.s.Event.fPending);
3464	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3465
3466	SVMEVENT Event;
3467	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3468
3469	uint8_t uVector = Event.n.u8Vector;
3470	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3471
3472	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3473
3474	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3475	AssertRC(rc);
3476
3477	if (Event.n.u1ErrorCodeValid)
3478	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3479
3480	if ( enmTrapType == TRPM_TRAP
3481	&& uVector == X86_XCPT_PF)
3482	{
3483	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3484	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3485	}
3486	else if (enmTrapType == TRPM_SOFTWARE_INT)
3487	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3488	pVCpu->hm.s.Event.fPending = false;
3489	}
3490
3491
3492	/**
3493	* Checks if the guest (or nested-guest) has an interrupt shadow active right
3494	* now.
3495	*
3496	* @returns @c true if the interrupt shadow is active, @c false otherwise.
3497	* @param pVCpu The cross context virtual CPU structure.
3498	*
3499	* @remarks No-long-jump zone!!!
3500	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3501	*/
3502	static bool hmR0SvmIsIntrShadowActive(PVMCPUCC pVCpu)
3503	{
3504	/*
3505	* Instructions like STI and MOV SS inhibit interrupts till the next instruction
3506	* completes. Check if we should inhibit interrupts or clear any existing
3507	* interrupt inhibition.
3508	*/
3509	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3510	{
3511	if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
3512	{
3513	/*
3514	* We can clear the inhibit force flag as even if we go back to the recompiler
3515	* without executing guest code in AMD-V, the flag's condition to be cleared is
3516	* met and thus the cleared state is correct.
3517	*/
3518	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3519	return false;
3520	}
3521	return true;
3522	}
3523	return false;
3524	}
3525
3526
3527	/**
3528	* Sets the virtual interrupt intercept control in the VMCB.
3529	*
3530	* @param pVCpu The cross context virtual CPU structure.
3531	* @param pVmcb Pointer to the VM control block.
3532	*/
3533	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3534	{
3535	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3536
3537	/*
3538	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3539	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3540	* (updating ISR at the right time) and inject the interrupt.
3541	*
3542	* With AVIC is supported, we could make use of the asynchronously delivery without
3543	* #VMEXIT and we would be passing the AVIC page to SVM.
3544	*
3545	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3546	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3547	*/
3548	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3549	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3550	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3551	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3552	Log4(("Set VINTR intercept\n"));
3553	}
3554
3555
3556	/**
3557	* Clears the virtual interrupt intercept control in the VMCB as
3558	* we are figured the guest is unable process any interrupts
3559	* at this point of time.
3560	*
3561	* @param pVCpu The cross context virtual CPU structure.
3562	* @param pVmcb Pointer to the VM control block.
3563	*/
3564	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3565	{
3566	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3567
3568	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3569	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3570	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3571	{
3572	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3573	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3574	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3575	Log4(("Cleared VINTR intercept\n"));
3576	}
3577	}
3578
3579
3580	/**
3581	* Evaluates the event to be delivered to the guest and sets it as the pending
3582	* event.
3583	*
3584	* @returns Strict VBox status code.
3585	* @param pVCpu The cross context virtual CPU structure.
3586	* @param pSvmTransient Pointer to the SVM transient structure.
3587	*/
3588	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3589	{
3590	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3591	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3592	\| CPUMCTX_EXTRN_RFLAGS
3593	\| CPUMCTX_EXTRN_HM_SVM_INT_SHADOW
3594	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3595
3596	Assert(!pVCpu->hm.s.Event.fPending);
3597	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3598	Assert(pVmcb);
3599
3600	bool const fGif = CPUMGetGuestGif(pCtx);
3601	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3602	bool const fBlockNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3603
3604	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3605	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3606	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3607
3608	/** @todo SMI. SMIs take priority over NMIs. */
3609
3610	/*
3611	* Check if the guest or nested-guest can receive NMIs.
3612	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3613	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3614	*/
3615	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3616	&& !fBlockNmi)
3617	{
3618	if ( fGif
3619	&& !fIntShadow)
3620	{
3621	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3622	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3623	{
3624	Log4(("Intercepting NMI -> #VMEXIT\n"));
3625	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3626	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3627	}
3628	#endif
3629	Log4(("Setting NMI pending for injection\n"));
3630	SVMEVENT Event;
3631	Event.u = 0;
3632	Event.n.u1Valid = 1;
3633	Event.n.u8Vector = X86_XCPT_NMI;
3634	Event.n.u3Type = SVM_EVENT_NMI;
3635	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3636	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3637	}
3638	else if (!fGif)
3639	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3640	else if (!pSvmTransient->fIsNestedGuest)
3641	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3642	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3643	}
3644	/*
3645	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3646	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3647	* it from the APIC device.
3648	*
3649	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3650	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3651	* do that work when we execute nested-guest code esp. since all the required information
3652	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3653	* the virtual interrupt controller.
3654	*
3655	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3656	*/
3657	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3658	&& !pVCpu->hm.s.fSingleInstruction)
3659	{
3660	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u32 & X86_EFL_IF)
3661	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3662	if ( fGif
3663	&& !fBlockInt
3664	&& !fIntShadow)
3665	{
3666	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3667	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3668	{
3669	Log4(("Intercepting INTR -> #VMEXIT\n"));
3670	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3671	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3672	}
3673	#endif
3674	uint8_t u8Interrupt;
3675	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3676	if (RT_SUCCESS(rc))
3677	{
3678	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3679	SVMEVENT Event;
3680	Event.u = 0;
3681	Event.n.u1Valid = 1;
3682	Event.n.u8Vector = u8Interrupt;
3683	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3684	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3685	}
3686	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3687	{
3688	/*
3689	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3690	* updated eventually when the TPR is written by the guest.
3691	*/
3692	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3693	}
3694	else
3695	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3696	}
3697	else if (!fGif)
3698	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3699	else if (!pSvmTransient->fIsNestedGuest)
3700	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3701	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3702	}
3703
3704	return VINF_SUCCESS;
3705	}
3706
3707
3708	/**
3709	* Injects any pending events into the guest (or nested-guest).
3710	*
3711	* @param pVCpu The cross context virtual CPU structure.
3712	* @param pVmcb Pointer to the VM control block.
3713	*
3714	* @remarks Must only be called when we are guaranteed to enter
3715	* hardware-assisted SVM execution and not return to ring-3
3716	* prematurely.
3717	*/
3718	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3719	{
3720	Assert(!TRPMHasTrap(pVCpu));
3721	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3722
3723	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3724	#ifdef VBOX_STRICT
3725	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3726	bool const fGif = CPUMGetGuestGif(pCtx);
3727	bool fAllowInt = fGif;
3728	if (fGif)
3729	{
3730	/*
3731	* For nested-guests we have no way to determine if we're injecting a physical or
3732	* virtual interrupt at this point. Hence the partial verification below.
3733	*/
3734	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3735	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3736	else
3737	fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
3738	}
3739	#endif
3740
3741	if (pVCpu->hm.s.Event.fPending)
3742	{
3743	SVMEVENT Event;
3744	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3745	Assert(Event.n.u1Valid);
3746
3747	/*
3748	* Validate event injection pre-conditions.
3749	*/
3750	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3751	{
3752	Assert(fAllowInt);
3753	Assert(!fIntShadow);
3754	}
3755	else if (Event.n.u3Type == SVM_EVENT_NMI)
3756	{
3757	Assert(fGif);
3758	Assert(!fIntShadow);
3759	}
3760
3761	/*
3762	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3763	* do this only when we are surely going to inject the NMI as otherwise if we return
3764	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3765	* SVM R0.
3766	*
3767	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3768	* which will set the VMCS field after actually delivering the NMI which we read on
3769	* VM-exit to determine the state.
3770	*/
3771	if ( Event.n.u3Type == SVM_EVENT_NMI
3772	&& Event.n.u8Vector == X86_XCPT_NMI
3773	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3774	{
3775	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3776	}
3777
3778	/*
3779	* Inject it (update VMCB for injection by the hardware).
3780	*/
3781	Log4(("Injecting pending HM event\n"));
3782	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3783	pVCpu->hm.s.Event.fPending = false;
3784
3785	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3786	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3787	else
3788	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3789	}
3790	else
3791	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3792
3793	/*
3794	* We could have injected an NMI through IEM and continue guest execution using
3795	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3796	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3797	*/
3798	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3799	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3800
3801	/*
3802	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3803	*
3804	* For nested-guests: We need to update it too for the scenario where IEM executes
3805	* the nested-guest but execution later continues here with an interrupt shadow active.
3806	*/
3807	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3808	}
3809
3810
3811	/**
3812	* Reports world-switch error and dumps some useful debug info.
3813	*
3814	* @param pVCpu The cross context virtual CPU structure.
3815	* @param rcVMRun The return code from VMRUN (or
3816	* VERR_SVM_INVALID_GUEST_STATE for invalid
3817	* guest-state).
3818	*/
3819	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3820	{
3821	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3822	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3823	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3824
3825	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3826	{
3827	#ifdef VBOX_STRICT
3828	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3829	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3830	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3831	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3832	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3833	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3834	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3835	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3836	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3837	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3838	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3839	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3840
3841	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3842	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3843	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3844
3845	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3846	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3847	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3848	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3849	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3850	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3851	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3852	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3853	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3854	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3855	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3856	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3857
3858	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3859	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3860	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3861	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3862	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3863	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3864	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3865	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3866	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3867	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3868	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3869	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3870	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3871	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3872	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3873	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3874	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3875	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3876	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3877	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3878
3879	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3880
3881	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3882	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3883
3884	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3885	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3886	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3887	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3888	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3889	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3890	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3891	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3892	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3893	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3894	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3895	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3896	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3897	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3898	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3899	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3900	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3901	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3902	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3903	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3904
3905	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3906	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3907
3908	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3909	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3910	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3911	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3912
3913	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3914	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3915
3916	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3917	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3918	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3919	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3920
3921	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3922	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3923	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3924	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3925	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3926	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3927	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3928
3929	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3930	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3931	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3932	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3933
3934	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3935	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3936	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3937
3938	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3939	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3940	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3941	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3942	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3943	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3944	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3945	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3946	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3947	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3948	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3949	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3950
3951	NOREF(pVmcb);
3952	#endif /* VBOX_STRICT */
3953	}
3954	else
3955	Log4Func(("rcVMRun=%d\n", rcVMRun));
3956	}
3957
3958
3959	/**
3960	* Check per-VM and per-VCPU force flag actions that require us to go back to
3961	* ring-3 for one reason or another.
3962	*
3963	* @returns Strict VBox status code (information status code included).
3964	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3965	* ring-3.
3966	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3967	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3968	* interrupts)
3969	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3970	* all EMTs to be in ring-3.
3971	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3972	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3973	* to the EM loop.
3974	*
3975	* @param pVCpu The cross context virtual CPU structure.
3976	*/
3977	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3978	{
3979	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3980	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
3981
3982	/* Could happen as a result of longjump. */
3983	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3984	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3985
3986	/* Update pending interrupts into the APIC's IRR. */
3987	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3988	APICUpdatePendingInterrupts(pVCpu);
3989
3990	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3991	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3992	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3993	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3994	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3995	{
3996	/* Pending PGM C3 sync. */
3997	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3998	{
3999	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
4000	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4001	if (rc != VINF_SUCCESS)
4002	{
4003	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
4004	return rc;
4005	}
4006	}
4007
4008	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
4009	/* -XXX- what was that about single stepping? */
4010	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
4011	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4012	{
4013	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4014	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
4015	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
4016	return rc;
4017	}
4018
4019	/* Pending VM request packets, such as hardware interrupts. */
4020	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
4021	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
4022	{
4023	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
4024	Log4Func(("Pending VM request forcing us back to ring-3\n"));
4025	return VINF_EM_PENDING_REQUEST;
4026	}
4027
4028	/* Pending PGM pool flushes. */
4029	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4030	{
4031	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4032	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4033	return VINF_PGM_POOL_FLUSH_PENDING;
4034	}
4035
4036	/* Pending DMA requests. */
4037	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4038	{
4039	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4040	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4041	return VINF_EM_RAW_TO_R3;
4042	}
4043	}
4044
4045	return VINF_SUCCESS;
4046	}
4047
4048
4049	/**
4050	* Does the preparations before executing guest code in AMD-V.
4051	*
4052	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4053	* recompiler. We must be cautious what we do here regarding committing
4054	* guest-state information into the VMCB assuming we assuredly execute the guest
4055	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4056	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4057	* that the recompiler can (and should) use them when it resumes guest
4058	* execution. Otherwise such operations must be done when we can no longer
4059	* exit to ring-3.
4060	*
4061	* @returns Strict VBox status code (informational status codes included).
4062	* @retval VINF_SUCCESS if we can proceed with running the guest.
4063	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4064	*
4065	* @param pVCpu The cross context virtual CPU structure.
4066	* @param pSvmTransient Pointer to the SVM transient structure.
4067	*/
4068	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4069	{
4070	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4071
4072	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4073	if (pSvmTransient->fIsNestedGuest)
4074	{
4075	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4076	return VINF_EM_RESCHEDULE_REM;
4077	}
4078	#endif
4079
4080	/* Check force flag actions that might require us to go back to ring-3. */
4081	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4082	if (rc != VINF_SUCCESS)
4083	return rc;
4084
4085	if (TRPMHasTrap(pVCpu))
4086	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4087	else if (!pVCpu->hm.s.Event.fPending)
4088	{
4089	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4090	if ( rc != VINF_SUCCESS
4091	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4092	{
4093	/* If a nested-guest VM-exit occurred, bail. */
4094	if (pSvmTransient->fIsNestedGuest)
4095	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4096	return rc;
4097	}
4098	}
4099
4100	/*
4101	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4102	* Just do it in software, see @bugref{8411}.
4103	* NB: If we could continue a task switch exit we wouldn't need to do this.
4104	*/
4105	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4106	if (RT_UNLIKELY( !g_fHmSvmFeatures
4107	&& pVCpu->hm.s.Event.fPending
4108	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4109	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4110
4111	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4112	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4113	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4114	#endif
4115
4116	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4117	/*
4118	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4119	*/
4120	if (pSvmTransient->fIsNestedGuest)
4121	hmR0SvmSetupVmcbNested(pVCpu);
4122	#endif
4123
4124	/*
4125	* Export the guest state bits that are not shared with the host in any way as we can
4126	* longjmp or get preempted in the midst of exporting some of the state.
4127	*/
4128	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4129	AssertRCReturn(rc, rc);
4130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4131
4132	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4133	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4134
4135	/*
4136	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4137	* world-switch so we can update it on the way back if the guest changed the TPR.
4138	*/
4139	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4140	{
4141	Assert(!pSvmTransient->fIsNestedGuest);
4142	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4143	if (pVM->hm.s.fTprPatchingActive)
4144	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4145	else
4146	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4147	}
4148
4149	/*
4150	* No longjmps to ring-3 from this point on!!!
4151	*
4152	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4153	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4154	*/
4155	VMMRZCallRing3Disable(pVCpu);
4156
4157	/*
4158	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4159	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4160	* preemption disabled for a while. Since this is purly to aid the
4161	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4162	* disable interrupt on NT.
4163	*
4164	* We need to check for force-flags that could've possible been altered since we last
4165	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4166	* see @bugref{6398}).
4167	*
4168	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4169	* to ring-3 before executing guest code.
4170	*/
4171	pSvmTransient->fEFlags = ASMIntDisableFlags();
4172	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4173	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4174	{
4175	ASMSetFlags(pSvmTransient->fEFlags);
4176	VMMRZCallRing3Enable(pVCpu);
4177	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4178	return VINF_EM_RAW_TO_R3;
4179	}
4180	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4181	{
4182	ASMSetFlags(pSvmTransient->fEFlags);
4183	VMMRZCallRing3Enable(pVCpu);
4184	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4185	return VINF_EM_RAW_INTERRUPT;
4186	}
4187
4188	return VINF_SUCCESS;
4189	}
4190
4191
4192	/**
4193	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4194	* doing so.
4195	*
4196	* This means there is no backing out to ring-3 or anywhere else at this point.
4197	*
4198	* @param pVCpu The cross context virtual CPU structure.
4199	* @param pSvmTransient Pointer to the SVM transient structure.
4200	*
4201	* @remarks Called with preemption disabled.
4202	* @remarks No-long-jump zone!!!
4203	*/
4204	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4205	{
4206	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4207	Assert(VMMR0IsLogFlushDisabled(pVCpu));
4208	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4209
4210	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4211	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4212
4213	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4214	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4215
4216	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4217
4218	if (!CPUMIsGuestFPUStateActive(pVCpu))
4219	{
4220	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4221	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4222	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4223	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4224	}
4225
4226	/* Load the state shared between host and guest (FPU, debug). */
4227	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4228	hmR0SvmExportSharedState(pVCpu, pVmcb);
4229
4230	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4231	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4232
4233	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4234	RTCPUID const idHostCpu = pHostCpu->idCpu;
4235	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4236
4237	/* Setup TSC offsetting. */
4238	if ( pSvmTransient->fUpdateTscOffsetting
4239	\|\| fMigratedHostCpu)
4240	{
4241	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4242	pSvmTransient->fUpdateTscOffsetting = false;
4243	}
4244
4245	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4246	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4247	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4248	else
4249	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4250
4251	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4252	if (fMigratedHostCpu)
4253	pVmcb->ctrl.u32VmcbCleanBits = 0;
4254
4255	/* Store status of the shared guest-host state at the time of VMRUN. */
4256	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4257	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4258
4259	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4260	uint8_t *pbMsrBitmap;
4261	if (!pSvmTransient->fIsNestedGuest)
4262	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4263	else
4264	{
4265	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4266
4267	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4268	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4269	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4270	}
4271	#else
4272	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4273	#endif
4274
4275	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4276	/* Flush the appropriate tagged-TLB entries. */
4277	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4278	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4279
4280	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4281
4282	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4283	to start executing. */
4284
4285	/*
4286	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4287	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4288	*
4289	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4290	*/
4291	if ( pVM->cpum.ro.HostFeatures.fRdTscP
4292	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4293	{
4294	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4295	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4296	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4297	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4298	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4299	pSvmTransient->fRestoreTscAuxMsr = true;
4300	}
4301	else
4302	{
4303	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4304	pSvmTransient->fRestoreTscAuxMsr = false;
4305	}
4306	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4307
4308	/*
4309	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4310	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4311	* from the VMCB.
4312	*/
4313	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4314	if (!fSupportsVmcbCleanBits)
4315	pVmcb->ctrl.u32VmcbCleanBits = 0;
4316	}
4317
4318
4319	/**
4320	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4321	*
4322	* @returns VBox strict status code.
4323	* @param pVCpu The cross context virtual CPU structure.
4324	* @param HCPhysVmcb The host physical address of the VMCB.
4325	*
4326	* @remarks No-long-jump zone!!!
4327	*/
4328	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4329	{
4330	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4331	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4332	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4333	}
4334
4335
4336	/**
4337	* Performs some essential restoration of state after running guest (or
4338	* nested-guest) code in AMD-V.
4339	*
4340	* @param pVCpu The cross context virtual CPU structure.
4341	* @param pSvmTransient Pointer to the SVM transient structure.
4342	* @param rcVMRun Return code of VMRUN.
4343	*
4344	* @remarks Called with interrupts disabled.
4345	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4346	* unconditionally when it is safe to do so.
4347	*/
4348	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4349	{
4350	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4351
4352	uint64_t const uHostTsc = ASMReadTSC(); /* Read the TSC as soon as possible. */
4353	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4354	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4355
4356	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4357	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4358
4359	/* TSC read must be done early for maximum accuracy. */
4360	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4361	{
4362	if (!pSvmTransient->fIsNestedGuest)
4363	TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVmcbCtrl->u64TSCOffset);
4364	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4365	else
4366	{
4367	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4368	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uHostTsc + pVmcbCtrl->u64TSCOffset);
4369	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4370	}
4371	#endif
4372	}
4373
4374	if (pSvmTransient->fRestoreTscAuxMsr)
4375	{
4376	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4377	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4378	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4379	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4380	}
4381
4382	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4383	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4384	TMNotifyEndOfExecution(pVM, pVCpu); /* Notify TM that the guest is no longer running. */
4385	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4386
4387	Assert(!(ASMGetFlags() & X86_EFL_IF));
4388	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4389	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4390
4391	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4392	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4393	{
4394	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4395	return;
4396	}
4397
4398	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4399	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4400	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4401	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4402
4403	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4404	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4405	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4406	#else
4407	/*
4408	* Always import the following:
4409	*
4410	* - RIP for exit optimizations and evaluating event injection on re-entry.
4411	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4412	* state on preemption.
4413	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4414	* - CS for exit optimizations.
4415	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4416	* assembly switcher code.
4417	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4418	*/
4419	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4420	\| CPUMCTX_EXTRN_RFLAGS
4421	\| CPUMCTX_EXTRN_RAX
4422	\| CPUMCTX_EXTRN_RSP
4423	\| CPUMCTX_EXTRN_CS
4424	\| CPUMCTX_EXTRN_HWVIRT
4425	\| CPUMCTX_EXTRN_HM_SVM_INT_SHADOW
4426	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4427	\| HMSVM_CPUMCTX_SHARED_STATE);
4428	#endif
4429
4430	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4431	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4432	{
4433	Assert(!pSvmTransient->fIsNestedGuest);
4434	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4435	if ( pVM->hm.s.fTprPatchingActive
4436	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4437	{
4438	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4439	AssertRC(rc);
4440	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4441	}
4442	/* Sync TPR when we aren't intercepting CR8 writes. */
4443	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4444	{
4445	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4446	AssertRC(rc);
4447	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4448	}
4449	}
4450
4451	#ifdef DEBUG_ramshankar
4452	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4453	{
4454	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4455	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4456	0 /* uVerbose */);
4457	}
4458	#endif
4459
4460	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4461	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4462	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
4463	}
4464
4465
4466	/**
4467	* Runs the guest code using AMD-V.
4468	*
4469	* @returns Strict VBox status code.
4470	* @param pVCpu The cross context virtual CPU structure.
4471	* @param pcLoops Pointer to the number of executed loops.
4472	*/
4473	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4474	{
4475	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4476	Assert(pcLoops);
4477	Assert(*pcLoops <= cMaxResumeLoops);
4478
4479	SVMTRANSIENT SvmTransient;
4480	RT_ZERO(SvmTransient);
4481	SvmTransient.fUpdateTscOffsetting = true;
4482	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4483
4484	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4485	for (;;)
4486	{
4487	Assert(!HMR0SuspendPending());
4488	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4489
4490	/* Preparatory work for running nested-guest code, this may force us to return to
4491	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4492	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4493	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4494	if (rc != VINF_SUCCESS)
4495	break;
4496
4497	/*
4498	* No longjmps to ring-3 from this point on!!!
4499	*
4500	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4501	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4502	*/
4503	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4504	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4505
4506	/* Restore any residual host-state and save any bits shared between host and guest
4507	into the guest-CPU state. Re-enables interrupts! */
4508	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4509
4510	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4511	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4512	{
4513	if (rc == VINF_SUCCESS)
4514	rc = VERR_SVM_INVALID_GUEST_STATE;
4515	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4516	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4517	break;
4518	}
4519
4520	/* Handle the #VMEXIT. */
4521	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4522	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4523	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4524	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4525	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4526	if (rc != VINF_SUCCESS)
4527	break;
4528	if (++(*pcLoops) >= cMaxResumeLoops)
4529	{
4530	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4531	rc = VINF_EM_RAW_INTERRUPT;
4532	break;
4533	}
4534	}
4535
4536	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4537	return rc;
4538	}
4539
4540
4541	/**
4542	* Runs the guest code using AMD-V in single step mode.
4543	*
4544	* @returns Strict VBox status code.
4545	* @param pVCpu The cross context virtual CPU structure.
4546	* @param pcLoops Pointer to the number of executed loops.
4547	*/
4548	static VBOXSTRICTRC hmR0SvmRunGuestCodeStep(PVMCPUCC pVCpu, uint32_t *pcLoops)
4549	{
4550	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4551	Assert(pcLoops);
4552	Assert(*pcLoops <= cMaxResumeLoops);
4553
4554	SVMTRANSIENT SvmTransient;
4555	RT_ZERO(SvmTransient);
4556	SvmTransient.fUpdateTscOffsetting = true;
4557	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4558
4559	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4560	uint16_t uCsStart = pCtx->cs.Sel;
4561	uint64_t uRipStart = pCtx->rip;
4562
4563	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4564	for (;;)
4565	{
4566	Assert(!HMR0SuspendPending());
4567	AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
4568	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
4569	(unsigned)RTMpCpuId(), *pcLoops));
4570
4571	/* Preparatory work for running nested-guest code, this may force us to return to
4572	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4573	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4574	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4575	if (rc != VINF_SUCCESS)
4576	break;
4577
4578	/*
4579	* No longjmps to ring-3 from this point on!!!
4580	*
4581	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4582	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4583	*/
4584	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4585
4586	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4587
4588	/* Restore any residual host-state and save any bits shared between host and guest
4589	into the guest-CPU state. Re-enables interrupts! */
4590	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4591
4592	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4593	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4594	{
4595	if (rc == VINF_SUCCESS)
4596	rc = VERR_SVM_INVALID_GUEST_STATE;
4597	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4598	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4599	return rc;
4600	}
4601
4602	/* Handle the #VMEXIT. */
4603	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4604	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4605	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4606	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4607	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4608	if (rc != VINF_SUCCESS)
4609	break;
4610	if (++(*pcLoops) >= cMaxResumeLoops)
4611	{
4612	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4613	rc = VINF_EM_RAW_INTERRUPT;
4614	break;
4615	}
4616
4617	/*
4618	* Did the RIP change, if so, consider it a single step.
4619	* Otherwise, make sure one of the TFs gets set.
4620	*/
4621	if ( pCtx->rip != uRipStart
4622	\|\| pCtx->cs.Sel != uCsStart)
4623	{
4624	rc = VINF_EM_DBG_STEPPED;
4625	break;
4626	}
4627	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR_MASK;
4628	}
4629
4630	/*
4631	* Clear the X86_EFL_TF if necessary.
4632	*/
4633	if (pVCpu->hmr0.s.fClearTrapFlag)
4634	{
4635	pVCpu->hmr0.s.fClearTrapFlag = false;
4636	pCtx->eflags.Bits.u1TF = 0;
4637	}
4638
4639	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4640	return rc;
4641	}
4642
4643	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4644	/**
4645	* Runs the nested-guest code using AMD-V.
4646	*
4647	* @returns Strict VBox status code.
4648	* @param pVCpu The cross context virtual CPU structure.
4649	* @param pcLoops Pointer to the number of executed loops. If we're switching
4650	* from the guest-code execution loop to this nested-guest
4651	* execution loop pass the remainder value, else pass 0.
4652	*/
4653	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4654	{
4655	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4656	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4657	Assert(pcLoops);
4658	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4659
4660	SVMTRANSIENT SvmTransient;
4661	RT_ZERO(SvmTransient);
4662	SvmTransient.fUpdateTscOffsetting = true;
4663	SvmTransient.pVmcb = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4664	SvmTransient.fIsNestedGuest = true;
4665
4666	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4667	for (;;)
4668	{
4669	Assert(!HMR0SuspendPending());
4670	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4671
4672	/* Preparatory work for running nested-guest code, this may force us to return to
4673	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4674	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4675	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4676	if ( rc != VINF_SUCCESS
4677	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4678	break;
4679
4680	/*
4681	* No longjmps to ring-3 from this point on!!!
4682	*
4683	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4684	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4685	*/
4686	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4687
4688	rc = hmR0SvmRunGuest(pVCpu, pCtx->hwvirt.svm.HCPhysVmcb);
4689
4690	/* Restore any residual host-state and save any bits shared between host and guest
4691	into the guest-CPU state. Re-enables interrupts! */
4692	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4693
4694	if (RT_LIKELY( rc == VINF_SUCCESS
4695	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4696	{ /* extremely likely */ }
4697	else
4698	{
4699	/* VMRUN failed, shouldn't really happen, Guru. */
4700	if (rc != VINF_SUCCESS)
4701	break;
4702
4703	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4704	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4705	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4706	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4707	break;
4708	}
4709
4710	/* Handle the #VMEXIT. */
4711	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4712	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4713	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pCtx->hwvirt.svm.CTX_SUFF(pVmcb));
4714	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4715	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4716	if (rc == VINF_SUCCESS)
4717	{
4718	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4719	{
4720	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4721	rc = VINF_SVM_VMEXIT;
4722	}
4723	else
4724	{
4725	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4726	continue;
4727	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4728	rc = VINF_EM_RAW_INTERRUPT;
4729	}
4730	}
4731	else
4732	Assert(rc != VINF_SVM_VMEXIT);
4733	break;
4734	/** @todo NSTSVM: handle single-stepping. */
4735	}
4736
4737	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4738	return rc;
4739	}
4740	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4741
4742
4743	/**
4744	* Runs the guest code using AMD-V.
4745	*
4746	* @returns Strict VBox status code.
4747	* @param pVCpu The cross context virtual CPU structure.
4748	*/
4749	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4750	{
4751	AssertPtr(pVCpu);
4752	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4753	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4754	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4755	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4756
4757	uint32_t cLoops = 0;
4758	VBOXSTRICTRC rc;
4759	for (;;)
4760	{
4761	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4762	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4763	#else
4764	NOREF(pCtx);
4765	bool const fInNestedGuestMode = false;
4766	#endif
4767	if (!fInNestedGuestMode)
4768	{
4769	if (!pVCpu->hm.s.fSingleInstruction)
4770	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4771	else
4772	rc = hmR0SvmRunGuestCodeStep(pVCpu, &cLoops);
4773	}
4774	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4775	else
4776	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4777
4778	if (rc == VINF_SVM_VMRUN)
4779	{
4780	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4781	continue;
4782	}
4783	if (rc == VINF_SVM_VMEXIT)
4784	{
4785	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4786	continue;
4787	}
4788	#endif
4789	break;
4790	}
4791
4792	/* Fixup error codes. */
4793	if (rc == VERR_EM_INTERPRETER)
4794	rc = VINF_EM_RAW_EMULATE_INSTR;
4795	else if (rc == VINF_EM_RESET)
4796	rc = VINF_EM_TRIPLE_FAULT;
4797
4798	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4799	rc = hmR0SvmExitToRing3(pVCpu, rc);
4800	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4801	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
4802	return rc;
4803	}
4804
4805
4806	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4807	/**
4808	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4809	*
4810	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4811	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4812	*/
4813	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4814	{
4815	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4816	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4817	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4818	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4819	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4820	const bool fRep = pIoExitInfo->n.u1Rep;
4821	const bool fStrIo = pIoExitInfo->n.u1Str;
4822
4823	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4824	NULL /* pIoExitInfo */);
4825	}
4826
4827
4828	/**
4829	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4830	* SVM_EXIT_INVALID).
4831	*
4832	* @returns VBox status code (informational status codes included).
4833	* @param pVCpu The cross context virtual CPU structure.
4834	* @param pSvmTransient Pointer to the SVM transient structure.
4835	*/
4836	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4837	{
4838	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4839	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4840	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4841
4842	/*
4843	* We import the complete state here because we use separate VMCBs for the guest and the
4844	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4845	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4846	*/
4847	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4848	do { \
4849	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4850	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4851	} while (0)
4852
4853	/*
4854	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4855	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4856	*/
4857	PSVMVMCB pVmcbNstGst = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pVmcb);
4858	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4859	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4860	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4861	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4862	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4863
4864	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4865	switch (uExitCode)
4866	{
4867	case SVM_EXIT_CPUID:
4868	{
4869	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4870	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4871	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4872	}
4873
4874	case SVM_EXIT_RDTSC:
4875	{
4876	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4877	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4878	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4879	}
4880
4881	case SVM_EXIT_RDTSCP:
4882	{
4883	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4884	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4885	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4886	}
4887
4888	case SVM_EXIT_MONITOR:
4889	{
4890	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4891	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4892	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4893	}
4894
4895	case SVM_EXIT_MWAIT:
4896	{
4897	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4898	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4899	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4900	}
4901
4902	case SVM_EXIT_HLT:
4903	{
4904	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4905	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4906	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4907	}
4908
4909	case SVM_EXIT_MSR:
4910	{
4911	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4912	{
4913	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4914	uint16_t offMsrpm;
4915	uint8_t uMsrpmBit;
4916	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4917	if (RT_SUCCESS(rc))
4918	{
4919	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4920	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4921
4922	uint8_t const pbMsrBitmap = (uint8_t const )pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvMsrBitmap);
4923	pbMsrBitmap += offMsrpm;
4924	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4925	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4926
4927	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4928	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4929	{
4930	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4931	}
4932	}
4933	else
4934	{
4935	/*
4936	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4937	* See AMD-V spec. "15.11 MSR Intercepts".
4938	*/
4939	Assert(rc == VERR_OUT_OF_RANGE);
4940	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4941	}
4942	}
4943	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4944	}
4945
4946	case SVM_EXIT_IOIO:
4947	{
4948	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4949	{
4950	void *pvIoBitmap = pVCpu->cpum.GstCtx.hwvirt.svm.CTX_SUFF(pvIoBitmap);
4951	SVMIOIOEXITINFO IoExitInfo;
4952	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4953	bool const fIntercept = hmR0SvmIsIoInterceptSet(pvIoBitmap, &IoExitInfo);
4954	if (fIntercept)
4955	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4956	}
4957	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4958	}
4959
4960	case SVM_EXIT_XCPT_PF:
4961	{
4962	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4963	if (pVM->hmr0.s.fNestedPaging)
4964	{
4965	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4966	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4967
4968	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4969	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4970	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4971
4972	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4973	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4974	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4975	return VINF_SUCCESS;
4976	}
4977	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4978	}
4979
4980	case SVM_EXIT_XCPT_UD:
4981	{
4982	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4983	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4984	hmR0SvmSetPendingXcptUD(pVCpu);
4985	return VINF_SUCCESS;
4986	}
4987
4988	case SVM_EXIT_XCPT_MF:
4989	{
4990	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4991	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4992	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4993	}
4994
4995	case SVM_EXIT_XCPT_DB:
4996	{
4997	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4998	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4999	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
5000	}
5001
5002	case SVM_EXIT_XCPT_AC:
5003	{
5004	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5005	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5006	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5007	}
5008
5009	case SVM_EXIT_XCPT_BP:
5010	{
5011	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5012	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5013	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5014	}
5015
5016	case SVM_EXIT_READ_CR0:
5017	case SVM_EXIT_READ_CR3:
5018	case SVM_EXIT_READ_CR4:
5019	{
5020	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5021	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5022	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5023	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5024	}
5025
5026	case SVM_EXIT_CR0_SEL_WRITE:
5027	{
5028	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5029	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5030	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5031	}
5032
5033	case SVM_EXIT_WRITE_CR0:
5034	case SVM_EXIT_WRITE_CR3:
5035	case SVM_EXIT_WRITE_CR4:
5036	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5037	{
5038	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5039	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5040
5041	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5042	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5043	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5044	}
5045
5046	case SVM_EXIT_PAUSE:
5047	{
5048	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5049	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5050	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5051	}
5052
5053	case SVM_EXIT_VINTR:
5054	{
5055	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5056	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5057	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5058	}
5059
5060	case SVM_EXIT_INTR:
5061	case SVM_EXIT_NMI:
5062	case SVM_EXIT_SMI:
5063	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5064	{
5065	/*
5066	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5067	*
5068	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5069	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5070	* nested-guest #VMEXIT.
5071	*
5072	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5073	* while trying to inject interrupts, see comment at the top of this function.
5074	*/
5075	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5076	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5077	}
5078
5079	case SVM_EXIT_FERR_FREEZE:
5080	{
5081	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5082	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5083	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5084	}
5085
5086	case SVM_EXIT_INVLPG:
5087	{
5088	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5089	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5090	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5091	}
5092
5093	case SVM_EXIT_WBINVD:
5094	{
5095	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5096	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5097	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5098	}
5099
5100	case SVM_EXIT_INVD:
5101	{
5102	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5103	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5104	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5105	}
5106
5107	case SVM_EXIT_RDPMC:
5108	{
5109	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5110	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5111	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5112	}
5113
5114	default:
5115	{
5116	switch (uExitCode)
5117	{
5118	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5119	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5120	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5121	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5122	{
5123	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5124	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5125	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5126	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5127	}
5128
5129	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5130	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5131	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5132	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5133	{
5134	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5135	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5136	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5137	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5138	}
5139
5140	case SVM_EXIT_XCPT_DE:
5141	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5142	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5143	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5144	case SVM_EXIT_XCPT_OF:
5145	case SVM_EXIT_XCPT_BR:
5146	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5147	case SVM_EXIT_XCPT_NM:
5148	case SVM_EXIT_XCPT_DF:
5149	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5150	case SVM_EXIT_XCPT_TS:
5151	case SVM_EXIT_XCPT_NP:
5152	case SVM_EXIT_XCPT_SS:
5153	case SVM_EXIT_XCPT_GP:
5154	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5155	case SVM_EXIT_XCPT_15: /* Reserved. */
5156	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5157	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5158	case SVM_EXIT_XCPT_MC:
5159	case SVM_EXIT_XCPT_XF:
5160	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5161	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5162	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5163	{
5164	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5165	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5166	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5167	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5168	}
5169
5170	case SVM_EXIT_XSETBV:
5171	{
5172	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5173	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5174	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5175	}
5176
5177	case SVM_EXIT_TASK_SWITCH:
5178	{
5179	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5180	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5181	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5182	}
5183
5184	case SVM_EXIT_IRET:
5185	{
5186	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5187	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5188	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5189	}
5190
5191	case SVM_EXIT_SHUTDOWN:
5192	{
5193	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5194	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5195	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5196	}
5197
5198	case SVM_EXIT_VMMCALL:
5199	{
5200	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5201	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5202	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5203	}
5204
5205	case SVM_EXIT_CLGI:
5206	{
5207	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5208	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5209	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5210	}
5211
5212	case SVM_EXIT_STGI:
5213	{
5214	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5215	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5216	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5217	}
5218
5219	case SVM_EXIT_VMLOAD:
5220	{
5221	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5222	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5223	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5224	}
5225
5226	case SVM_EXIT_VMSAVE:
5227	{
5228	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5229	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5230	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5231	}
5232
5233	case SVM_EXIT_INVLPGA:
5234	{
5235	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5236	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5237	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5238	}
5239
5240	case SVM_EXIT_VMRUN:
5241	{
5242	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5243	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5244	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5245	}
5246
5247	case SVM_EXIT_RSM:
5248	{
5249	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5250	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5251	hmR0SvmSetPendingXcptUD(pVCpu);
5252	return VINF_SUCCESS;
5253	}
5254
5255	case SVM_EXIT_SKINIT:
5256	{
5257	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5258	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5259	hmR0SvmSetPendingXcptUD(pVCpu);
5260	return VINF_SUCCESS;
5261	}
5262
5263	case SVM_EXIT_NPF:
5264	{
5265	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5266	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5267	}
5268
5269	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5270	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5271
5272	default:
5273	{
5274	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5275	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5276	return VERR_SVM_UNKNOWN_EXIT;
5277	}
5278	}
5279	}
5280	}
5281	/* not reached */
5282
5283	#undef NST_GST_VMEXIT_CALL_RET
5284	}
5285	#endif
5286
5287
5288	/**
5289	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5290	*
5291	* @returns Strict VBox status code (informational status codes included).
5292	* @param pVCpu The cross context virtual CPU structure.
5293	* @param pSvmTransient Pointer to the SVM transient structure.
5294	*/
5295	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5296	{
5297	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5298	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5299
5300	#ifdef DEBUG_ramshankar
5301	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5302	do { \
5303	if ((a_fDbg) == 1) \
5304	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5305	int rc = a_CallExpr; \
5306	if ((a_fDbg) == 1) \
5307	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5308	return rc; \
5309	} while (0)
5310	#else
5311	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5312	#endif
5313
5314	/*
5315	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5316	* for most guests under normal workloads (for some definition of "normal").
5317	*/
5318	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5319	switch (uExitCode)
5320	{
5321	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5322	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5323	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5324	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5325	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5326	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5327	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5328	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5329	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5330	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5331
5332	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5333	case SVM_EXIT_INTR:
5334	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5335
5336	case SVM_EXIT_READ_CR0:
5337	case SVM_EXIT_READ_CR3:
5338	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5339
5340	case SVM_EXIT_CR0_SEL_WRITE:
5341	case SVM_EXIT_WRITE_CR0:
5342	case SVM_EXIT_WRITE_CR3:
5343	case SVM_EXIT_WRITE_CR4:
5344	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5345
5346	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5347	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5348	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5349	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5350	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5351	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5352	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5353	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5354	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5355	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5356	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5357	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5358	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5359	case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5360	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5361	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5362
5363	default:
5364	{
5365	switch (pSvmTransient->u64ExitCode)
5366	{
5367	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5368	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5369	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5370	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5371	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5372
5373	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5374	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5375	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5376	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5377	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5378
5379	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5380	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5381
5382	case SVM_EXIT_SMI:
5383	case SVM_EXIT_INIT:
5384	{
5385	/*
5386	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5387	* If it ever does, we want to know about it so log the exit code and bail.
5388	*/
5389	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5390	}
5391
5392	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5393	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5394	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5395	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5396	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5397	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5398	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5399	#else
5400	case SVM_EXIT_CLGI:
5401	case SVM_EXIT_STGI:
5402	case SVM_EXIT_VMLOAD:
5403	case SVM_EXIT_VMSAVE:
5404	case SVM_EXIT_INVLPGA:
5405	case SVM_EXIT_VMRUN:
5406	#endif
5407	case SVM_EXIT_RSM:
5408	case SVM_EXIT_SKINIT:
5409	{
5410	hmR0SvmSetPendingXcptUD(pVCpu);
5411	return VINF_SUCCESS;
5412	}
5413
5414	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
5415	case SVM_EXIT_XCPT_DE:
5416	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5417	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5418	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5419	case SVM_EXIT_XCPT_OF:
5420	case SVM_EXIT_XCPT_BR:
5421	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5422	case SVM_EXIT_XCPT_NM:
5423	case SVM_EXIT_XCPT_DF:
5424	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5425	case SVM_EXIT_XCPT_TS:
5426	case SVM_EXIT_XCPT_NP:
5427	case SVM_EXIT_XCPT_SS:
5428	/* SVM_EXIT_XCPT_GP: / / Handled above. */
5429	/* SVM_EXIT_XCPT_PF: */
5430	case SVM_EXIT_XCPT_15: /* Reserved. */
5431	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5432	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5433	case SVM_EXIT_XCPT_MC:
5434	case SVM_EXIT_XCPT_XF:
5435	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5436	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5437	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5438	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5439	#endif /* HMSVM_ALWAYS_TRAP_ALL_XCPTS */
5440
5441	default:
5442	{
5443	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5444	pVCpu->hm.s.u32HMError = uExitCode;
5445	return VERR_SVM_UNKNOWN_EXIT;
5446	}
5447	}
5448	}
5449	}
5450	/* not reached */
5451	#undef VMEXIT_CALL_RET
5452	}
5453
5454
5455	#ifdef VBOX_STRICT
5456	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
5457	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
5458	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
5459
5460	# define HMSVM_ASSERT_PREEMPT_CPUID() \
5461	do \
5462	{ \
5463	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
5464	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
5465	} while (0)
5466
5467	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
5468	do { \
5469	AssertPtr((a_pVCpu)); \
5470	AssertPtr((a_pSvmTransient)); \
5471	Assert(ASMIntAreEnabled()); \
5472	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
5473	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
5474	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
5475	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
5476	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
5477	HMSVM_ASSERT_PREEMPT_CPUID(); \
5478	} while (0)
5479	#else
5480	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
5481	do { \
5482	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
5483	} while (0)
5484	#endif
5485
5486
5487	/**
5488	* Gets the IEM exception flags for the specified SVM event.
5489	*
5490	* @returns The IEM exception flags.
5491	* @param pEvent Pointer to the SVM event.
5492	*
5493	* @remarks This function currently only constructs flags required for
5494	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
5495	* and CR2 aspects of an exception are not included).
5496	*/
5497	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
5498	{
5499	uint8_t const uEventType = pEvent->n.u3Type;
5500	uint32_t fIemXcptFlags;
5501	switch (uEventType)
5502	{
5503	case SVM_EVENT_EXCEPTION:
5504	/*
5505	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
5506	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
5507	*/
5508	if (pEvent->n.u8Vector == X86_XCPT_BP)
5509	{
5510	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
5511	break;
5512	}
5513	if (pEvent->n.u8Vector == X86_XCPT_OF)
5514	{
5515	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
5516	break;
5517	}
5518	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
5519	RT_FALL_THRU();
5520	case SVM_EVENT_NMI:
5521	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5522	break;
5523
5524	case SVM_EVENT_EXTERNAL_IRQ:
5525	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5526	break;
5527
5528	case SVM_EVENT_SOFTWARE_INT:
5529	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5530	break;
5531
5532	default:
5533	fIemXcptFlags = 0;
5534	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
5535	break;
5536	}
5537	return fIemXcptFlags;
5538	}
5539
5540
5541	/**
5542	* Handle a condition that occurred while delivering an event through the guest
5543	* IDT.
5544	*
5545	* @returns VBox status code (informational error codes included).
5546	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
5547	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
5548	* continue execution of the guest which will delivery the \#DF.
5549	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5550	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5551	*
5552	* @param pVCpu The cross context virtual CPU structure.
5553	* @param pSvmTransient Pointer to the SVM transient structure.
5554	*
5555	* @remarks No-long-jump zone!!!
5556	*/
5557	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5558	{
5559	int rc = VINF_SUCCESS;
5560	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
5561	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
5562
5563	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
5564	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
5565	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
5566
5567	/*
5568	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
5569	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
5570	*
5571	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
5572	*/
5573	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
5574	{
5575	IEMXCPTRAISE enmRaise;
5576	IEMXCPTRAISEINFO fRaiseInfo;
5577	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
5578	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5579	if (fExitIsHwXcpt)
5580	{
5581	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
5582	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
5583	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5584	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5585	}
5586	else
5587	{
5588	/*
5589	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
5590	* then we end up here.
5591	*
5592	* If the event was:
5593	* - a software interrupt, we can re-execute the instruction which will
5594	* regenerate the event.
5595	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
5596	* - a hardware exception or external interrupt, we re-inject it.
5597	*/
5598	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5599	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
5600	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5601	else
5602	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5603	}
5604
5605	switch (enmRaise)
5606	{
5607	case IEMXCPTRAISE_CURRENT_XCPT:
5608	case IEMXCPTRAISE_PREV_EVENT:
5609	{
5610	/* For software interrupts, we shall re-execute the instruction. */
5611	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
5612	{
5613	RTGCUINTPTR GCPtrFaultAddress = 0;
5614
5615	/* If we are re-injecting an NMI, clear NMI blocking. */
5616	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
5617	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5618
5619	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
5620	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5621	{
5622	pSvmTransient->fVectoringPF = true;
5623	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
5624	pVCpu->cpum.GstCtx.cr2));
5625	}
5626	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
5627	&& uIdtVector == X86_XCPT_PF)
5628	{
5629	/*
5630	* If the previous exception was a #PF, we need to recover the CR2 value.
5631	* This can't happen with shadow paging.
5632	*/
5633	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
5634	}
5635
5636	/*
5637	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
5638	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
5639	*/
5640	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
5641	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
5642	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
5643
5644	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
5645	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
5646	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
5647	}
5648	break;
5649	}
5650
5651	case IEMXCPTRAISE_REEXEC_INSTR:
5652	{
5653	Assert(rc == VINF_SUCCESS);
5654	break;
5655	}
5656
5657	case IEMXCPTRAISE_DOUBLE_FAULT:
5658	{
5659	/*
5660	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
5661	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
5662	* converted into a #DF.
5663	*/
5664	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5665	{
5666	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
5667	pSvmTransient->fVectoringDoublePF = true;
5668	Assert(rc == VINF_SUCCESS);
5669	}
5670	else
5671	{
5672	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
5673	hmR0SvmSetPendingXcptDF(pVCpu);
5674	rc = VINF_HM_DOUBLE_FAULT;
5675	}
5676	break;
5677	}
5678
5679	case IEMXCPTRAISE_TRIPLE_FAULT:
5680	{
5681	rc = VINF_EM_RESET;
5682	break;
5683	}
5684
5685	case IEMXCPTRAISE_CPU_HANG:
5686	{
5687	rc = VERR_EM_GUEST_CPU_HANG;
5688	break;
5689	}
5690
5691	default:
5692	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
5693	}
5694	}
5695	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
5696	return rc;
5697	}
5698
5699
5700	/**
5701	* Advances the guest RIP by the number of bytes specified in @a cb.
5702	*
5703	* @param pVCpu The cross context virtual CPU structure.
5704	* @param cb RIP increment value in bytes.
5705	*/
5706	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
5707	{
5708	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5709	pCtx->rip += cb;
5710
5711	/* Update interrupt shadow. */
5712	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5713	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
5714	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5715	}
5716
5717
5718	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5719	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
5720	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5721
5722	/** @name \#VMEXIT handlers.
5723	* @{
5724	*/
5725
5726	/**
5727	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
5728	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
5729	*/
5730	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5731	{
5732	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
5733
5734	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
5735	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
5736	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
5737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
5738
5739	/*
5740	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
5741	* signal -before- the timer fires if the current interrupt is our own timer or a some
5742	* other host interrupt. We also cannot examine what interrupt it is until the host
5743	* actually take the interrupt.
5744	*
5745	* Going back to executing guest code here unconditionally causes random scheduling
5746	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
5747	*/
5748	return VINF_EM_RAW_INTERRUPT;
5749	}
5750
5751
5752	/**
5753	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
5754	*/
5755	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5756	{
5757	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
5758
5759	VBOXSTRICTRC rcStrict;
5760	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
5761	if (fSupportsNextRipSave)
5762	{
5763	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
5764	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
5765	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
5766	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
5767	}
5768	else
5769	{
5770	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
5771	rcStrict = IEMExecOne(pVCpu);
5772	}
5773
5774	if (rcStrict == VINF_IEM_RAISED_XCPT)
5775	{
5776	rcStrict = VINF_SUCCESS;
5777	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
5778	}
5779	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
5780	return rcStrict;
5781	}
5782
5783
5784	/**
5785	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
5786	*/
5787	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5788	{
5789	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
5790
5791	VBOXSTRICTRC rcStrict;
5792	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
5793	if (fSupportsNextRipSave)
5794	{
5795	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
5796	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
5797	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
5798	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
5799	}
5800	else
5801	{
5802	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
5803	rcStrict = IEMExecOne(pVCpu);
5804	}
5805
5806	if (rcStrict == VINF_IEM_RAISED_XCPT)
5807	{
5808	rcStrict = VINF_SUCCESS;
5809	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
5810	}
5811	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
5812	return rcStrict;
5813	}
5814
5815
5816	/**
5817	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
5818	*/
5819	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5820	{
5821	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
5822
5823	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
5824	VBOXSTRICTRC rcStrict;
5825	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
5826	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
5827	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
5828	if (!pExitRec)
5829	{
5830	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
5831	if (fSupportsNextRipSave)
5832	{
5833	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
5834	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
5835	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
5836	}
5837	else
5838	{
5839	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
5840	rcStrict = IEMExecOne(pVCpu);
5841	}
5842
5843	if (rcStrict == VINF_IEM_RAISED_XCPT)
5844	{
5845	rcStrict = VINF_SUCCESS;
5846	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
5847	}
5848	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
5849	}
5850	else
5851	{
5852	/*
5853	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
5854	*/
5855	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
5856
5857	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
5858	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
5859
5860	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
5861
5862	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
5863	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
5864	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
5865	}
5866	return rcStrict;
5867	}
5868
5869
5870	/**
5871	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
5872	*/
5873	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5874	{
5875	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
5876
5877	VBOXSTRICTRC rcStrict;
5878	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
5879	if (fSupportsNextRipSave)
5880	{
5881	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
5882	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
5883	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
5884	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
5885	}
5886	else
5887	{
5888	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
5889	rcStrict = IEMExecOne(pVCpu);
5890	}
5891
5892	if (rcStrict == VINF_SUCCESS)
5893	pSvmTransient->fUpdateTscOffsetting = true;
5894	else if (rcStrict == VINF_IEM_RAISED_XCPT)
5895	{
5896	rcStrict = VINF_SUCCESS;
5897	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
5898	}
5899	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
5900	return rcStrict;
5901	}
5902
5903
5904	/**
5905	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
5906	*/
5907	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5908	{
5909	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
5910
5911	VBOXSTRICTRC rcStrict;
5912	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
5913	if (fSupportsNextRipSave)
5914	{
5915	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
5916	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
5917	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
5918	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
5919	}
5920	else
5921	{
5922	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
5923	rcStrict = IEMExecOne(pVCpu);
5924	}
5925
5926	if (rcStrict == VINF_SUCCESS)
5927	pSvmTransient->fUpdateTscOffsetting = true;
5928	else if (rcStrict == VINF_IEM_RAISED_XCPT)
5929	{
5930	rcStrict = VINF_SUCCESS;
5931	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
5932	}
5933	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
5934	return rcStrict;
5935	}
5936
5937
5938	/**
5939	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
5940	*/
5941	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5942	{
5943	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
5944
5945	VBOXSTRICTRC rcStrict;
5946	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
5947	if (fSupportsNextRipSave)
5948	{
5949	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
5950	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
5951	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
5952	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
5953	}
5954	else
5955	{
5956	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
5957	rcStrict = IEMExecOne(pVCpu);
5958	}
5959
5960	if (rcStrict == VINF_IEM_RAISED_XCPT)
5961	{
5962	rcStrict = VINF_SUCCESS;
5963	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
5964	}
5965	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
5966	return rcStrict;
5967	}
5968
5969
5970	/**
5971	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
5972	*/
5973	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5974	{
5975	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
5976	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5977
5978	VBOXSTRICTRC rcStrict;
5979	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
5980	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
5981	if ( fSupportsDecodeAssists
5982	&& fSupportsNextRipSave)
5983	{
5984	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
5985	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
5986	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
5987	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
5988	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
5989	}
5990	else
5991	{
5992	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
5993	rcStrict = IEMExecOne(pVCpu);
5994	}
5995
5996	if (rcStrict == VINF_IEM_RAISED_XCPT)
5997	{
5998	rcStrict = VINF_SUCCESS;
5999	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6000	}
6001	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6002	return VBOXSTRICTRC_VAL(rcStrict);
6003	}
6004
6005
6006	/**
6007	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
6008	*/
6009	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6010	{
6011	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6012
6013	VBOXSTRICTRC rcStrict;
6014	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6015	if (fSupportsNextRipSave)
6016	{
6017	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6018	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6019	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6020	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
6021	}
6022	else
6023	{
6024	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6025	rcStrict = IEMExecOne(pVCpu);
6026	}
6027
6028	if ( rcStrict == VINF_EM_HALT
6029	\|\| rcStrict == VINF_SUCCESS)
6030	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
6031	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6032	{
6033	rcStrict = VINF_SUCCESS;
6034	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6035	}
6036	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6037	if (rcStrict != VINF_SUCCESS)
6038	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
6039	return VBOXSTRICTRC_VAL(rcStrict);;
6040	}
6041
6042
6043	/**
6044	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
6045	*/
6046	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6047	{
6048	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6049
6050	/*
6051	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
6052	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
6053	* segment override prefix or other prefixes might be used, in which case we fallback to
6054	* IEMExecOne() to figure out.
6055	*/
6056	VBOXSTRICTRC rcStrict;
6057	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6058	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
6059	if (cbInstr)
6060	{
6061	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
6062	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
6063	}
6064	else
6065	{
6066	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6067	rcStrict = IEMExecOne(pVCpu);
6068	}
6069
6070	if (rcStrict == VINF_IEM_RAISED_XCPT)
6071	{
6072	rcStrict = VINF_SUCCESS;
6073	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6074	}
6075	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6076	return rcStrict;
6077	}
6078
6079
6080	/**
6081	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
6082	*/
6083	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6084	{
6085	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6086
6087	VBOXSTRICTRC rcStrict;
6088	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6089	if (fSupportsNextRipSave)
6090	{
6091	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6092	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6093	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6094	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
6095	}
6096	else
6097	{
6098	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6099	rcStrict = IEMExecOne(pVCpu);
6100	}
6101
6102	if ( rcStrict == VINF_EM_HALT
6103	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
6104	rcStrict = VINF_SUCCESS;
6105	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6106	{
6107	rcStrict = VINF_SUCCESS;
6108	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6109	}
6110	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6111	return rcStrict;
6112	}
6113
6114
6115	/**
6116	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
6117	* \#VMEXIT.
6118	*/
6119	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6120	{
6121	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6122	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6123	return VINF_EM_RESET;
6124	}
6125
6126
6127	/**
6128	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
6129	*/
6130	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6131	{
6132	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6133	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6134	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
6135	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
6136	RT_NOREF(pVmcb);
6137	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6138	return VERR_SVM_UNEXPECTED_EXIT;
6139	}
6140
6141
6142	/**
6143	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
6144	*/
6145	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6146	{
6147	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6148
6149	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6150	Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6151	#ifdef VBOX_WITH_STATISTICS
6152	switch (pSvmTransient->u64ExitCode)
6153	{
6154	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
6155	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
6156	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
6157	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
6158	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
6159	}
6160	#endif
6161
6162	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
6163	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6164	if ( fSupportsDecodeAssists
6165	&& fSupportsNextRipSave)
6166	{
6167	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6168	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6169	if (fMovCRx)
6170	{
6171	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
6172	\| CPUMCTX_EXTRN_APIC_TPR);
6173	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6174	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
6175	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6176	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6177	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6178	return VBOXSTRICTRC_VAL(rcStrict);
6179	}
6180	/* else: SMSW instruction, fall back below to IEM for this. */
6181	}
6182
6183	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6184	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6185	AssertMsg( rcStrict == VINF_SUCCESS
6186	\|\| rcStrict == VINF_PGM_SYNC_CR3
6187	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
6188	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6189	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
6190	if (rcStrict == VINF_IEM_RAISED_XCPT)
6191	{
6192	rcStrict = VINF_SUCCESS;
6193	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6194	}
6195	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6196	return rcStrict;
6197	}
6198
6199
6200	/**
6201	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
6202	*/
6203	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6204	{
6205	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6206
6207	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6208	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
6209	Assert(iCrReg <= 15);
6210
6211	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
6212	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6213	bool fDecodedInstr = false;
6214	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
6215	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6216	if ( fSupportsDecodeAssists
6217	&& fSupportsNextRipSave)
6218	{
6219	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6220	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6221	if (fMovCRx)
6222	{
6223	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
6224	\| CPUMCTX_EXTRN_APIC_TPR);
6225	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6226	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6227	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
6228	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6229	fDecodedInstr = true;
6230	}
6231	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
6232	}
6233
6234	if (!fDecodedInstr)
6235	{
6236	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6237	Log4Func(("iCrReg=%#x\n", iCrReg));
6238	rcStrict = IEMExecOne(pVCpu);
6239	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
6240	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
6241	rcStrict = VERR_EM_INTERPRETER;
6242	}
6243
6244	if (rcStrict == VINF_SUCCESS)
6245	{
6246	switch (iCrReg)
6247	{
6248	case 0:
6249	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
6250	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
6251	break;
6252
6253	case 2:
6254	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
6255	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
6256	break;
6257
6258	case 3:
6259	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
6260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
6261	break;
6262
6263	case 4:
6264	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
6265	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
6266	break;
6267
6268	case 8:
6269	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6270	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
6271	break;
6272
6273	default:
6274	{
6275	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
6276	pSvmTransient->u64ExitCode, iCrReg));
6277	break;
6278	}
6279	}
6280	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6281	}
6282	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6283	{
6284	rcStrict = VINF_SUCCESS;
6285	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6286	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6287	}
6288	else
6289	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
6290	return rcStrict;
6291	}
6292
6293
6294	/**
6295	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
6296	*
6297	* @returns Strict VBox status code.
6298	* @param pVCpu The cross context virtual CPU structure.
6299	* @param pVmcb Pointer to the VM control block.
6300	*/
6301	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
6302	{
6303	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
6304	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
6305
6306	VBOXSTRICTRC rcStrict;
6307	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6308	if (fSupportsNextRipSave)
6309	{
6310	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
6311	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
6312	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
6313	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6314	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
6315	}
6316	else
6317	{
6318	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
6319	rcStrict = IEMExecOne(pVCpu);
6320	}
6321
6322	AssertMsg( rcStrict == VINF_SUCCESS
6323	\|\| rcStrict == VINF_IEM_RAISED_XCPT
6324	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
6325	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6326
6327	if (rcStrict == VINF_IEM_RAISED_XCPT)
6328	{
6329	rcStrict = VINF_SUCCESS;
6330	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6331	}
6332	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6333	return rcStrict;
6334	}
6335
6336
6337	/**
6338	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
6339	*
6340	* @returns Strict VBox status code.
6341	* @param pVCpu The cross context virtual CPU structure.
6342	* @param pVmcb Pointer to the VM control block.
6343	* @param pSvmTransient Pointer to the SVM-transient structure.
6344	*/
6345	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
6346	{
6347	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6348	uint32_t const idMsr = pCtx->ecx;
6349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
6350	Log4Func(("idMsr=%#RX32\n", idMsr));
6351
6352	/*
6353	* Handle TPR patching MSR writes.
6354	* We utilitize the LSTAR MSR for patching.
6355	*/
6356	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6357	if ( idMsr == MSR_K8_LSTAR
6358	&& pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
6359	{
6360	unsigned cbInstr;
6361	if (fSupportsNextRipSave)
6362	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6363	else
6364	{
6365	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
6366	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
6367	if ( rc == VINF_SUCCESS
6368	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
6369	Assert(cbInstr > 0);
6370	else
6371	cbInstr = 0;
6372	}
6373
6374	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
6375	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
6376	{
6377	int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
6378	AssertRCReturn(rc, rc);
6379	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6380	}
6381
6382	int rc = VINF_SUCCESS;
6383	hmR0SvmAdvanceRip(pVCpu, cbInstr);
6384	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6385	return rc;
6386	}
6387
6388	/*
6389	* Handle regular MSR writes.
6390	*/
6391	VBOXSTRICTRC rcStrict;
6392	if (fSupportsNextRipSave)
6393	{
6394	/** @todo Optimize this: We don't need to get much of the MSR state here
6395	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
6396	* clear the applicable extern flags. */
6397	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
6398	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6399	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
6400	}
6401	else
6402	{
6403	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
6404	rcStrict = IEMExecOne(pVCpu);
6405	}
6406
6407	AssertMsg( rcStrict == VINF_SUCCESS
6408	\|\| rcStrict == VINF_IEM_RAISED_XCPT
6409	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
6410	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6411
6412	if (rcStrict == VINF_SUCCESS)
6413	{
6414	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
6415	if ( idMsr >= MSR_IA32_X2APIC_START
6416	&& idMsr <= MSR_IA32_X2APIC_END)
6417	{
6418	/*
6419	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
6420	* When full APIC register virtualization is implemented we'll have to make sure
6421	* APIC state is saved from the VMCB before IEM changes it.
6422	*/
6423	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6424	}
6425	else
6426	{
6427	switch (idMsr)
6428	{
6429	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
6430	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
6431	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
6432	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
6433	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
6434	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
6435	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
6436	}
6437	}
6438	}
6439	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6440	{
6441	rcStrict = VINF_SUCCESS;
6442	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6443	}
6444	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6445	return rcStrict;
6446	}
6447
6448
6449	/**
6450	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
6451	* \#VMEXIT.
6452	*/
6453	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6454	{
6455	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6456
6457	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6458	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
6459	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
6460
6461	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
6462	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
6463	}
6464
6465
6466	/**
6467	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
6468	*/
6469	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6470	{
6471	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6472	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6473
6474	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
6475
6476	/** @todo Stepping with nested-guest. */
6477	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6478	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
6479	{
6480	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
6481	if (pSvmTransient->fWasGuestDebugStateActive)
6482	{
6483	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
6484	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6485	return VERR_SVM_UNEXPECTED_EXIT;
6486	}
6487
6488	/*
6489	* Lazy DR0-3 loading.
6490	*/
6491	if (!pSvmTransient->fWasHyperDebugStateActive)
6492	{
6493	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
6494	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
6495
6496	/* Don't intercept DRx read and writes. */
6497	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6498	pVmcb->ctrl.u16InterceptRdDRx = 0;
6499	pVmcb->ctrl.u16InterceptWrDRx = 0;
6500	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
6501
6502	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6503	VMMRZCallRing3Disable(pVCpu);
6504	HM_DISABLE_PREEMPT(pVCpu);
6505
6506	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
6507	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
6508	Assert(CPUMIsGuestDebugStateActive(pVCpu));
6509
6510	HM_RESTORE_PREEMPT();
6511	VMMRZCallRing3Enable(pVCpu);
6512
6513	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
6514	return VINF_SUCCESS;
6515	}
6516	}
6517
6518	/*
6519	* Interpret the read/writing of DRx.
6520	*/
6521	/** @todo Decode assist. */
6522	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6523	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
6524	if (RT_LIKELY(rc == VINF_SUCCESS))
6525	{
6526	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
6527	/** @todo CPUM should set this flag! */
6528	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
6529	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6530	}
6531	else
6532	Assert(rc == VERR_EM_INTERPRETER);
6533	return rc;
6534	}
6535
6536
6537	/**
6538	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
6539	*/
6540	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6541	{
6542	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6543	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
6544	VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
6545	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
6546	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
6547	return rc;
6548	}
6549
6550
6551	/**
6552	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
6553	*/
6554	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6555	{
6556	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6557	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6558
6559	/** @todo decode assists... */
6560	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6561	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
6562	{
6563	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6564	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6565	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
6566	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
6567	{
6568	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
6569	hmR0SvmUpdateVmRunFunction(pVCpu);
6570	}
6571	}
6572	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6573	{
6574	rcStrict = VINF_SUCCESS;
6575	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6576	}
6577	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6578	return rcStrict;
6579	}
6580
6581
6582	/**
6583	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
6584	*/
6585	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6586	{
6587	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6588	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
6589
6590	/* I/O operation lookup arrays. */
6591	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
6592	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
6593	the result (in AL/AX/EAX). */
6594	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6595	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6596	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6597
6598	Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6599
6600	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
6601	SVMIOIOEXITINFO IoExitInfo;
6602	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
6603	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
6604	uint32_t cbValue = s_aIOSize[uIOWidth];
6605	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
6606
6607	if (RT_UNLIKELY(!cbValue))
6608	{
6609	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
6610	return VERR_EM_INTERPRETER;
6611	}
6612
6613	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
6614	VBOXSTRICTRC rcStrict;
6615	PCEMEXITREC pExitRec = NULL;
6616	if ( !pVCpu->hm.s.fSingleInstruction
6617	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
6618	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6619	!IoExitInfo.n.u1Str
6620	? IoExitInfo.n.u1Type == SVM_IOIO_READ
6621	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
6622	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
6623	: IoExitInfo.n.u1Type == SVM_IOIO_READ
6624	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
6625	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
6626	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6627	if (!pExitRec)
6628	{
6629	bool fUpdateRipAlready = false;
6630	if (IoExitInfo.n.u1Str)
6631	{
6632	/* INS/OUTS - I/O String instruction. */
6633	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
6634	* in EXITINFO1? Investigate once this thing is up and running. */
6635	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
6636	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
6637	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
6638	static IEMMODE const s_aenmAddrMode[8] =
6639	{
6640	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
6641	};
6642	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
6643	if (enmAddrMode != (IEMMODE)-1)
6644	{
6645	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
6646	if (cbInstr <= 15 && cbInstr >= 1)
6647	{
6648	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
6649	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6650	{
6651	/* Don't know exactly how to detect whether u3Seg is valid, currently
6652	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
6653	2384 Opterons when only checking NRIP. */
6654	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6655	if ( fSupportsNextRipSave
6656	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
6657	{
6658	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
6659	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
6660	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
6661	IoExitInfo.n.u3Seg, true /fIoChecked/);
6662	}
6663	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
6664	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
6665	X86_SREG_DS, true /fIoChecked/);
6666	else
6667	rcStrict = IEMExecOne(pVCpu);
6668	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
6669	}
6670	else
6671	{
6672	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
6673	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
6674	true /fIoChecked/);
6675	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
6676	}
6677	}
6678	else
6679	{
6680	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
6681	rcStrict = IEMExecOne(pVCpu);
6682	}
6683	}
6684	else
6685	{
6686	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
6687	rcStrict = IEMExecOne(pVCpu);
6688	}
6689	fUpdateRipAlready = true;
6690	}
6691	else
6692	{
6693	/* IN/OUT - I/O instruction. */
6694	Assert(!IoExitInfo.n.u1Rep);
6695
6696	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
6697	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6698	{
6699	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
6700	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
6701	&& !pCtx->eflags.Bits.u1TF)
6702	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
6703	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
6704	}
6705	else
6706	{
6707	uint32_t u32Val = 0;
6708	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
6709	if (IOM_SUCCESS(rcStrict))
6710	{
6711	/* Save result of I/O IN instr. in AL/AX/EAX. */
6712	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
6713	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
6714	}
6715	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
6716	&& !pCtx->eflags.Bits.u1TF)
6717	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
6718
6719	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
6720	}
6721	}
6722
6723	if (IOM_SUCCESS(rcStrict))
6724	{
6725	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
6726	if (!fUpdateRipAlready)
6727	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
6728
6729	/*
6730	* If any I/O breakpoints are armed, we need to check if one triggered
6731	* and take appropriate action.
6732	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
6733	*/
6734	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
6735	* execution engines about whether hyper BPs and such are pending. */
6736	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
6737	uint32_t const uDr7 = pCtx->dr[7];
6738	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
6739	&& X86_DR7_ANY_RW_IO(uDr7)
6740	&& (pCtx->cr4 & X86_CR4_DE))
6741	\|\| DBGFBpIsHwIoArmed(pVM)))
6742	{
6743	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6744	VMMRZCallRing3Disable(pVCpu);
6745	HM_DISABLE_PREEMPT(pVCpu);
6746
6747	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
6748	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
6749
6750	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
6751	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
6752	{
6753	/* Raise #DB. */
6754	pVmcb->guest.u64DR6 = pCtx->dr[6];
6755	pVmcb->guest.u64DR7 = pCtx->dr[7];
6756	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
6757	hmR0SvmSetPendingXcptDB(pVCpu);
6758	}
6759	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
6760	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
6761	else if ( rcStrict2 != VINF_SUCCESS
6762	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
6763	rcStrict = rcStrict2;
6764	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
6765
6766	HM_RESTORE_PREEMPT();
6767	VMMRZCallRing3Enable(pVCpu);
6768	}
6769
6770	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6771	}
6772
6773	#ifdef VBOX_STRICT
6774	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
6775	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
6776	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
6777	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
6778	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
6779	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
6780	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
6781	else
6782	{
6783	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
6784	* statuses, that the VMM device and some others may return. See
6785	* IOM_SUCCESS() for guidance. */
6786	AssertMsg( RT_FAILURE(rcStrict)
6787	\|\| rcStrict == VINF_SUCCESS
6788	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
6789	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
6790	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
6791	\|\| rcStrict == VINF_EM_DBG_STEPPED
6792	\|\| rcStrict == VINF_EM_RAW_TO_R3
6793	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED
6794	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6795	}
6796	#endif
6797	}
6798	else
6799	{
6800	/*
6801	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6802	*/
6803	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6804	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
6805	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
6806	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
6807	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
6808	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
6809	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
6810
6811	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6812	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6813
6814	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6815	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6816	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6817	}
6818	return rcStrict;
6819	}
6820
6821
6822	/**
6823	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
6824	*/
6825	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6826	{
6827	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6828	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6829	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
6830
6831	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6832	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6833	Assert(pVM->hmr0.s.fNestedPaging);
6834
6835	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
6836	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6837	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
6838	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
6839	thus intentionally not copied into u32ErrCode. */
6840
6841	Log4Func(("#NPF at CS:RIP=%04x:%#RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
6842	u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
6843
6844	/*
6845	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
6846	*/
6847	if ( pVM->hm.s.fTprPatchingAllowed
6848	&& (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
6849	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
6850	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
6851	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
6852	&& !CPUMIsGuestInLongModeEx(pCtx)
6853	&& !CPUMGetGuestCPL(pVCpu)
6854	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
6855	{
6856	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
6857	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6858
6859	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
6860	{
6861	/* Only attempt to patch the instruction once. */
6862	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
6863	if (!pPatch)
6864	return VINF_EM_HM_PATCH_TPR_INSTR;
6865	}
6866	}
6867
6868	/*
6869	* Determine the nested paging mode.
6870	*/
6871	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
6872	PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
6873
6874	/*
6875	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
6876	*/
6877	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
6878	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
6879	{
6880	/*
6881	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
6882	* injecting the original pending event would most likely cause the same MMIO #NPF.
6883	*/
6884	if (pVCpu->hm.s.Event.fPending)
6885	{
6886	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
6887	return VINF_EM_RAW_INJECT_TRPM_EVENT;
6888	}
6889
6890	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6891	VBOXSTRICTRC rcStrict;
6892	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6893	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
6894	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6895	if (!pExitRec)
6896	{
6897
6898	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
6899	u32ErrCode);
6900
6901	/*
6902	* If we succeed, resume guest execution.
6903	*
6904	* If we fail in interpreting the instruction because we couldn't get the guest
6905	* physical address of the page containing the instruction via the guest's page
6906	* tables (we would invalidate the guest page in the host TLB), resume execution
6907	* which would cause a guest page fault to let the guest handle this weird case.
6908	*
6909	* See @bugref{6043}.
6910	*/
6911	if ( rcStrict == VINF_SUCCESS
6912	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
6913	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
6914	{
6915	/* Successfully handled MMIO operation. */
6916	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
6917	rcStrict = VINF_SUCCESS;
6918	}
6919	}
6920	else
6921	{
6922	/*
6923	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6924	*/
6925	Assert(pCtx == &pVCpu->cpum.GstCtx);
6926	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6927	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
6928	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
6929
6930	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6931	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6932
6933	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6934	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6935	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6936	}
6937	return rcStrict;
6938	}
6939
6940	/*
6941	* Nested page-fault.
6942	*/
6943	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
6944	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
6945	TRPMResetTrap(pVCpu);
6946
6947	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%#RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
6948
6949	/*
6950	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
6951	*/
6952	if ( rc == VINF_SUCCESS
6953	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
6954	\|\| rc == VERR_PAGE_NOT_PRESENT)
6955	{
6956	/* We've successfully synced our shadow page tables. */
6957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
6958	rc = VINF_SUCCESS;
6959	}
6960
6961	/*
6962	* If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
6963	* re-inject the original event.
6964	*/
6965	if (pVCpu->hm.s.Event.fPending)
6966	{
6967	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
6968
6969	/*
6970	* If the #NPF handler requested emulation of the instruction, ignore it.
6971	* We need to re-inject the original event so as to not lose it.
6972	* Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
6973	* LiveCD is broken for other reasons).
6974	*/
6975	if (rc == VINF_EM_RAW_EMULATE_INSTR)
6976	rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
6977	}
6978
6979	return rc;
6980	}
6981
6982
6983	/**
6984	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
6985	* \#VMEXIT.
6986	*/
6987	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6988	{
6989	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6990	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
6991
6992	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
6993	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6994	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
6995
6996	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
6997	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6998	return VINF_SUCCESS;
6999	}
7000
7001
7002	/**
7003	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
7004	* \#VMEXIT.
7005	*/
7006	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7007	{
7008	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7009	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7010
7011	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
7012	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7013	#endif
7014
7015	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
7016	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
7017	{
7018	/*
7019	* AMD-V provides us with the exception which caused the TS; we collect
7020	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
7021	*/
7022	Log4Func(("TS occurred during event delivery\n"));
7023	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7024	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7025	}
7026
7027	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7028	* emulation. */
7029	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7030	return VERR_EM_INTERPRETER;
7031	}
7032
7033
7034	/**
7035	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7036	*/
7037	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7038	{
7039	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7040	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7041
7042	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7043	if (pVM->hm.s.fTprPatchingAllowed)
7044	{
7045	int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
7046	if (rc != VERR_NOT_FOUND)
7047	{
7048	Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
7049	return rc;
7050	}
7051	}
7052
7053	if (EMAreHypercallInstructionsEnabled(pVCpu))
7054	{
7055	unsigned cbInstr;
7056	if (hmR0SvmSupportsNextRipSave(pVCpu))
7057	{
7058	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7059	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7060	}
7061	else
7062	{
7063	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
7064	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
7065	if ( rc == VINF_SUCCESS
7066	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
7067	Assert(cbInstr > 0);
7068	else
7069	cbInstr = 0;
7070	}
7071
7072	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
7073	if (RT_SUCCESS(rcStrict))
7074	{
7075	/* Only update the RIP if we're continuing guest execution and not in the case
7076	of say VINF_GIM_R3_HYPERCALL. */
7077	if (rcStrict == VINF_SUCCESS)
7078	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7079
7080	return VBOXSTRICTRC_VAL(rcStrict);
7081	}
7082	else
7083	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
7084	}
7085
7086	hmR0SvmSetPendingXcptUD(pVCpu);
7087	return VINF_SUCCESS;
7088	}
7089
7090
7091	/**
7092	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7093	*/
7094	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7095	{
7096	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7097
7098	unsigned cbInstr;
7099	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7100	if (fSupportsNextRipSave)
7101	{
7102	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7103	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7104	}
7105	else
7106	{
7107	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
7108	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
7109	if ( rc == VINF_SUCCESS
7110	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
7111	Assert(cbInstr > 0);
7112	else
7113	cbInstr = 0;
7114	}
7115
7116	/** @todo The guest has likely hit a contended spinlock. We might want to
7117	* poke a schedule different guest VCPU. */
7118	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7119	return VINF_EM_RAW_INTERRUPT;
7120	}
7121
7122
7123	/**
7124	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
7125	* \#VMEXIT.
7126	*/
7127	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7128	{
7129	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7130	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
7131	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
7132
7133	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
7134	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
7135	}
7136
7137
7138	/**
7139	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
7140	*/
7141	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7142	{
7143	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7144
7145	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
7146	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7147	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
7148
7149	/* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
7150	* deliver a pending NMI right after. If the IRET faults, an NMI can come before the
7151	* handler executes. Yes, x86 is ugly.
7152	*/
7153	return VINF_EM_RAW_EMULATE_INSTR;
7154	}
7155
7156
7157	/**
7158	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
7159	* Conditional \#VMEXIT.
7160	*/
7161	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7162	{
7163	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7164	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7165	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7166
7167	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7168	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7169	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7170	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7171	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
7172	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7173
7174	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
7175	if (pVM->hmr0.s.fNestedPaging)
7176	{
7177	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7178	if ( !pSvmTransient->fVectoringDoublePF
7179	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7180	{
7181	/* A genuine guest #PF, reflect it to the guest. */
7182	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
7183	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
7184	uFaultAddress, uErrCode));
7185	}
7186	else
7187	{
7188	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7189	hmR0SvmSetPendingXcptDF(pVCpu);
7190	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
7191	}
7192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7193	return VINF_SUCCESS;
7194	}
7195	#endif
7196
7197	Assert(!pVM->hmr0.s.fNestedPaging);
7198
7199	/*
7200	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
7201	*/
7202	if ( pVM->hm.s.fTprPatchingAllowed
7203	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
7204	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
7205	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7206	&& !CPUMIsGuestInLongModeEx(pCtx)
7207	&& !CPUMGetGuestCPL(pVCpu)
7208	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7209	{
7210	RTGCPHYS GCPhysApicBase;
7211	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7212	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7213
7214	/* Check if the page at the fault-address is the APIC base. */
7215	RTGCPHYS GCPhysPage;
7216	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL /* pfFlags */, &GCPhysPage);
7217	if ( rc2 == VINF_SUCCESS
7218	&& GCPhysPage == GCPhysApicBase)
7219	{
7220	/* Only attempt to patch the instruction once. */
7221	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7222	if (!pPatch)
7223	return VINF_EM_HM_PATCH_TPR_INSTR;
7224	}
7225	}
7226
7227	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7228	pCtx->rip, uErrCode, pCtx->cr3));
7229
7230	/*
7231	* If it's a vectoring #PF, emulate injecting the original event injection as
7232	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
7233	* event injection that caused a #PF. See @bugref{6607}.
7234	*/
7235	if (pSvmTransient->fVectoringPF)
7236	{
7237	Assert(pVCpu->hm.s.Event.fPending);
7238	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7239	}
7240
7241	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
7242	int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7243
7244	Log4Func(("#PF: rc=%Rrc\n", rc));
7245
7246	if (rc == VINF_SUCCESS)
7247	{
7248	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7249	TRPMResetTrap(pVCpu);
7250	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7251	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7252	return rc;
7253	}
7254
7255	if (rc == VINF_EM_RAW_GUEST_TRAP)
7256	{
7257	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7258
7259	/*
7260	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
7261	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
7262	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
7263	*/
7264	if ( !pSvmTransient->fVectoringDoublePF
7265	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7266	{
7267	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
7268	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7269	TRPMResetTrap(pVCpu);
7270
7271	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7272	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
7273	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7274	&& CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
7275	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
7276	#endif
7277
7278	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
7279	}
7280	else
7281	{
7282	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7283	TRPMResetTrap(pVCpu);
7284	hmR0SvmSetPendingXcptDF(pVCpu);
7285	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
7286	}
7287
7288	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7289	return VINF_SUCCESS;
7290	}
7291
7292	TRPMResetTrap(pVCpu);
7293	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7294	return rc;
7295	}
7296
7297
7298	/**
7299	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
7300	* Conditional \#VMEXIT.
7301	*/
7302	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7303	{
7304	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7305	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
7306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7307
7308	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7309	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7310	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7311
7312	/** @todo if we accumulate more optional stuff here, we ought to combine the
7313	* reading of opcode bytes to avoid doing more than once. */
7314
7315	VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
7316	if (pVCpu->hm.s.fGIMTrapXcptUD)
7317	{
7318	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7319	uint8_t cbInstr = 0;
7320	rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
7321	if (rcStrict == VINF_SUCCESS)
7322	{
7323	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
7324	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7325	rcStrict = VINF_SUCCESS;
7326	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7327	}
7328	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
7329	rcStrict = VINF_SUCCESS;
7330	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
7331	rcStrict = VINF_GIM_R3_HYPERCALL;
7332	else
7333	{
7334	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
7335	rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
7336	}
7337	}
7338
7339	if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
7340	{
7341	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
7342	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
7343	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
7344	{
7345	/* Ideally, IEM should just handle all these special #UD situations, but
7346	we don't quite trust things to behave optimially when doing that. So,
7347	for now we'll restrict ourselves to a handful of possible sysenter and
7348	sysexit encodings that we filter right here. */
7349	uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
7350	uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
7351	uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
7352	uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
7353	RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
7354	if (cbInstr < cbMin \|\| cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
7355	{
7356	cbInstr = cbMin;
7357	int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
7358	AssertRCStmt(rc2, cbInstr = 0);
7359	}
7360	else
7361	memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
7362	if ( cbInstr == 0 /* read error */
7363	\|\| (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
7364	\|\| ( uCpl == 0
7365	&& ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
7366	\|\| ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
7367	&& (abInstr[0] & (X86_OP_REX_W \| 0xf0)) == X86_OP_REX_W))))
7368	{
7369	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
7370	\| CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
7371	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
7372	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), GCPtrInstr, abInstr, cbInstr);
7373	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
7374	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
7375	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
7376	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7377	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
7378	if (rcStrict == VINF_IEM_RAISED_XCPT)
7379	rcStrict = VINF_SUCCESS;
7380	return rcStrict;
7381	}
7382	Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
7383	}
7384	else
7385	Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7386	}
7387
7388	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
7389	if (RT_FAILURE(rcStrict))
7390	{
7391	hmR0SvmSetPendingXcptUD(pVCpu);
7392	rcStrict = VINF_SUCCESS;
7393	}
7394
7395	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7396	return rcStrict;
7397	}
7398
7399
7400	/**
7401	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
7402	* Conditional \#VMEXIT.
7403	*/
7404	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7405	{
7406	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7407	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7408	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
7409
7410	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7411	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7412
7413	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7414	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7415
7416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
7417
7418	if (!(pCtx->cr0 & X86_CR0_NE))
7419	{
7420	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7421	PDISSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
7422	unsigned cbInstr;
7423	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbInstr);
7424	if (RT_SUCCESS(rc))
7425	{
7426	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
7427	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
7428	if (RT_SUCCESS(rc))
7429	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7430	}
7431	else
7432	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
7433	return rc;
7434	}
7435
7436	hmR0SvmSetPendingXcptMF(pVCpu);
7437	return VINF_SUCCESS;
7438	}
7439
7440
7441	/**
7442	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
7443	* \#VMEXIT.
7444	*/
7445	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7446	{
7447	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7448	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7449	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7450	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
7451
7452	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7453	{
7454	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7455	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7456	}
7457
7458	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
7459
7460	/*
7461	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
7462	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
7463	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
7464	*/
7465	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7466	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7467	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7468	int rc = DBGFTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
7469	if (rc == VINF_EM_RAW_GUEST_TRAP)
7470	{
7471	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
7472	if (CPUMIsHyperDebugStateActive(pVCpu))
7473	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
7474
7475	/* Reflect the exception back to the guest. */
7476	hmR0SvmSetPendingXcptDB(pVCpu);
7477	rc = VINF_SUCCESS;
7478	}
7479
7480	/*
7481	* Update DR6.
7482	*/
7483	if (CPUMIsHyperDebugStateActive(pVCpu))
7484	{
7485	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
7486	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
7487	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7488	}
7489	else
7490	{
7491	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
7492	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
7493	}
7494
7495	return rc;
7496	}
7497
7498
7499	/**
7500	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
7501	* Conditional \#VMEXIT.
7502	*/
7503	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7504	{
7505	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7506	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7507	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
7508
7509	SVMEVENT Event;
7510	Event.u = 0;
7511	Event.n.u1Valid = 1;
7512	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7513	Event.n.u8Vector = X86_XCPT_AC;
7514	Event.n.u1ErrorCodeValid = 1;
7515	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7516	return VINF_SUCCESS;
7517	}
7518
7519
7520	/**
7521	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
7522	* Conditional \#VMEXIT.
7523	*/
7524	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7525	{
7526	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7527	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7528	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7529	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
7530
7531	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7532	int rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
7533	if (rc == VINF_EM_RAW_GUEST_TRAP)
7534	{
7535	SVMEVENT Event;
7536	Event.u = 0;
7537	Event.n.u1Valid = 1;
7538	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7539	Event.n.u8Vector = X86_XCPT_BP;
7540	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7541	rc = VINF_SUCCESS;
7542	}
7543
7544	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
7545	return rc;
7546	}
7547
7548
7549	/**
7550	* Hacks its way around the lovely mesa driver's backdoor accesses.
7551	*
7552	* @sa hmR0VmxHandleMesaDrvGp
7553	*/
7554	static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
7555	{
7556	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK);
7557	Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
7558	pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
7559	RT_NOREF(pCtx, pVmcb);
7560
7561	/* For now we'll just skip the instruction. */
7562	hmR0SvmAdvanceRip(pVCpu, 1);
7563	return VINF_SUCCESS;
7564	}
7565
7566
7567	/**
7568	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
7569	* backdoor logging w/o checking what it is running inside.
7570	*
7571	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
7572	* backdoor port and magic numbers loaded in registers.
7573	*
7574	* @returns true if it is, false if it isn't.
7575	* @sa hmR0VmxIsMesaDrvGp
7576	*/
7577	DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
7578	{
7579	/* Check magic and port. */
7580	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
7581	/Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));/
7582	if (pCtx->dx != UINT32_C(0x5658))
7583	return false;
7584	if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
7585	return false;
7586
7587	/* Check that it is #GP(0). */
7588	if (pVmcb->ctrl.u64ExitInfo1 != 0)
7589	return false;
7590
7591	/* Flat ring-3 CS. */
7592	/Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));/
7593	if (pVmcb->guest.u8CPL != 3)
7594	return false;
7595	if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
7596	return false;
7597
7598	/* 0xed: IN eAX,dx */
7599	if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
7600	{
7601	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_GPRS_MASK
7602	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
7603	uint8_t abInstr[1];
7604	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
7605	/Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); /
7606	if (RT_FAILURE(rc))
7607	return false;
7608	if (abInstr[0] != 0xed)
7609	return false;
7610	}
7611	else
7612	{
7613	/Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));/
7614	if (pVmcb->ctrl.abInstr[0] != 0xed)
7615	return false;
7616	}
7617	return true;
7618	}
7619
7620
7621	/**
7622	* \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
7623	* Conditional \#VMEXIT.
7624	*/
7625	HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7626	{
7627	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7628	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7629	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
7630
7631	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7632	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
7633
7634	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7635	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
7636	\|\| !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
7637	{
7638	SVMEVENT Event;
7639	Event.u = 0;
7640	Event.n.u1Valid = 1;
7641	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7642	Event.n.u8Vector = X86_XCPT_GP;
7643	Event.n.u1ErrorCodeValid = 1;
7644	Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7645	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7646	return VINF_SUCCESS;
7647	}
7648	return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
7649	}
7650
7651
7652	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM)
7653	/**
7654	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
7655	*/
7656	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7657	{
7658	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7659	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7660
7661	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7662	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
7663	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
7664	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
7665	Assert(uVector <= X86_XCPT_LAST);
7666	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
7667
7668	SVMEVENT Event;
7669	Event.u = 0;
7670	Event.n.u1Valid = 1;
7671	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7672	Event.n.u8Vector = uVector;
7673	switch (uVector)
7674	{
7675	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
7676	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
7677	case X86_XCPT_DF:
7678	case X86_XCPT_TS:
7679	case X86_XCPT_NP:
7680	case X86_XCPT_SS:
7681	case X86_XCPT_GP:
7682	case X86_XCPT_AC:
7683	{
7684	Event.n.u1ErrorCodeValid = 1;
7685	Event.n.u32ErrorCode = uErrCode;
7686	break;
7687	}
7688	}
7689
7690	#ifdef VBOX_WITH_STATISTICS
7691	switch (uVector)
7692	{
7693	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
7694	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
7695	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
7696	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
7697	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
7698	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
7699	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
7700	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
7701	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
7702	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
7703	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
7704	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
7705	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
7706	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
7707	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
7708	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
7709	default:
7710	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7711	break;
7712	}
7713	#endif
7714
7715	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7716	return VINF_SUCCESS;
7717	}
7718	#endif
7719
7720	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7721	/**
7722	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
7723	*/
7724	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7725	{
7726	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7727
7728	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7729	Assert(pVmcb);
7730	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
7731
7732	VBOXSTRICTRC rcStrict;
7733	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7734	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
7735	if (fSupportsNextRipSave)
7736	{
7737	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
7738	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7739	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
7740	}
7741	else
7742	{
7743	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
7744	rcStrict = IEMExecOne(pVCpu);
7745	}
7746
7747	if (rcStrict == VINF_SUCCESS)
7748	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
7749	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7750	{
7751	rcStrict = VINF_SUCCESS;
7752	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7753	}
7754	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7755	return rcStrict;
7756	}
7757
7758
7759	/**
7760	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
7761	*/
7762	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7763	{
7764	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7765
7766	/*
7767	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
7768	* we only intercept STGI when events are pending for GIF to become 1.
7769	*/
7770	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7771	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
7772	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
7773
7774	VBOXSTRICTRC rcStrict;
7775	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7776	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
7777	if (fSupportsNextRipSave)
7778	{
7779	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
7780	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7781	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
7782	}
7783	else
7784	{
7785	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
7786	rcStrict = IEMExecOne(pVCpu);
7787	}
7788
7789	if (rcStrict == VINF_SUCCESS)
7790	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
7791	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7792	{
7793	rcStrict = VINF_SUCCESS;
7794	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7795	}
7796	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7797	return rcStrict;
7798	}
7799
7800
7801	/**
7802	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
7803	*/
7804	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7805	{
7806	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7807
7808	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7809	Assert(pVmcb);
7810	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7811
7812	VBOXSTRICTRC rcStrict;
7813	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7814	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
7815	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
7816	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
7817	if (fSupportsNextRipSave)
7818	{
7819	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
7820	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7821	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
7822	}
7823	else
7824	{
7825	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
7826	rcStrict = IEMExecOne(pVCpu);
7827	}
7828
7829	if (rcStrict == VINF_SUCCESS)
7830	{
7831	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
7832	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
7833	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
7834	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
7835	}
7836	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7837	{
7838	rcStrict = VINF_SUCCESS;
7839	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7840	}
7841	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7842	return rcStrict;
7843	}
7844
7845
7846	/**
7847	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
7848	*/
7849	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7850	{
7851	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7852
7853	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7854	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7855
7856	VBOXSTRICTRC rcStrict;
7857	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7858	if (fSupportsNextRipSave)
7859	{
7860	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7861	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7862	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
7863	}
7864	else
7865	{
7866	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7867	rcStrict = IEMExecOne(pVCpu);
7868	}
7869
7870	if (rcStrict == VINF_IEM_RAISED_XCPT)
7871	{
7872	rcStrict = VINF_SUCCESS;
7873	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7874	}
7875	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7876	return rcStrict;
7877	}
7878
7879
7880	/**
7881	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
7882	*/
7883	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7884	{
7885	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7886
7887	VBOXSTRICTRC rcStrict;
7888	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7889	if (fSupportsNextRipSave)
7890	{
7891	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7892	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7893	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7894	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
7895	}
7896	else
7897	{
7898	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7899	rcStrict = IEMExecOne(pVCpu);
7900	}
7901
7902	if (rcStrict == VINF_IEM_RAISED_XCPT)
7903	{
7904	rcStrict = VINF_SUCCESS;
7905	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7906	}
7907	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7908	return rcStrict;
7909	}
7910
7911
7912	/**
7913	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
7914	*/
7915	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7916	{
7917	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7918	/* We shall import the entire state here, just in case we enter and continue execution of
7919	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
7920	could lose lose part of CPU state. */
7921	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7922
7923	VBOXSTRICTRC rcStrict;
7924	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7925	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
7926	if (fSupportsNextRipSave)
7927	{
7928	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7929	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7930	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
7931	}
7932	else
7933	{
7934	/* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
7935	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
7936	instruction itself, see @bugref{7243#c126} */
7937	rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), NULL /* pcbWritten */);
7938	}
7939	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
7940
7941	if (rcStrict == VINF_SUCCESS)
7942	{
7943	rcStrict = VINF_SVM_VMRUN;
7944	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
7945	}
7946	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7947	{
7948	rcStrict = VINF_SUCCESS;
7949	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7950	}
7951	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7952	return rcStrict;
7953	}
7954
7955
7956	/**
7957	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
7958	* Unconditional \#VMEXIT.
7959	*/
7960	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7961	{
7962	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7963	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7964
7965	if (pVCpu->hm.s.Event.fPending)
7966	{
7967	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7968	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7969	}
7970
7971	hmR0SvmSetPendingXcptDB(pVCpu);
7972	return VINF_SUCCESS;
7973	}
7974
7975
7976	/**
7977	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
7978	* Conditional \#VMEXIT.
7979	*/
7980	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7981	{
7982	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7983	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7984
7985	SVMEVENT Event;
7986	Event.u = 0;
7987	Event.n.u1Valid = 1;
7988	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7989	Event.n.u8Vector = X86_XCPT_BP;
7990	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7991	return VINF_SUCCESS;
7992	}
7993	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
7994
7995	/** @} */
7996

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 87700

Download in other formats: