HMSVMR0.cpp@ 98187

Last change on this file since 98187 was 98103, checked in by vboxsync, 2 years ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 380.4 KB

Line
1	/* $Id: HMSVMR0.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_HM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/thread.h>
36
37	#include <VBox/vmm/pdmapi.h>
38	#include <VBox/vmm/dbgf.h>
39	#include <VBox/vmm/iem.h>
40	#include <VBox/vmm/iom.h>
41	#include <VBox/vmm/tm.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/gcm.h>
44	#include <VBox/vmm/gim.h>
45	#include <VBox/vmm/apic.h>
46	#include "HMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include <VBox/err.h>
49	#include "HMSVMR0.h"
50	#include "dtrace/VBoxVMM.h"
51
52	#ifdef DEBUG_ramshankar
53	# define HMSVM_SYNC_FULL_GUEST_STATE
54	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
55	# define HMSVM_ALWAYS_TRAP_PF
56	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
57	#endif
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	#ifdef VBOX_WITH_STATISTICS
64	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
66	if ((u64ExitCode) == SVM_EXIT_NPF) \
67	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
68	else \
69	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
70	} while (0)
71
72	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
74	if ((u64ExitCode) == SVM_EXIT_NPF) \
75	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
76	else \
77	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
78	} while (0)
79
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
81	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
82	if ((u64ExitCode) == SVM_EXIT_NPF) \
83	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
84	else \
85	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
86	} while (0)
87	#else
88	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
89	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
90	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
91	#endif /* !VBOX_WITH_STATISTICS */
92
93	/** If we decide to use a function table approach this can be useful to
94	* switch to a "static DECLCALLBACK(int)". */
95	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
96
97	/**
98	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
99	* guest using hardware-assisted SVM.
100	*
101	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
102	* are swapped and restored across the world-switch and also registers like
103	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
104	* \#VMEXIT.
105	*/
106	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
107	\| CPUMCTX_EXTRN_RFLAGS \
108	\| CPUMCTX_EXTRN_RAX \
109	\| CPUMCTX_EXTRN_RSP \
110	\| CPUMCTX_EXTRN_SREG_MASK \
111	\| CPUMCTX_EXTRN_CR0 \
112	\| CPUMCTX_EXTRN_CR2 \
113	\| CPUMCTX_EXTRN_CR3 \
114	\| CPUMCTX_EXTRN_TABLE_MASK \
115	\| CPUMCTX_EXTRN_DR6 \
116	\| CPUMCTX_EXTRN_DR7 \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_HWVIRT \
121	\| CPUMCTX_EXTRN_INHIBIT_INT \
122	\| CPUMCTX_EXTRN_HM_SVM_MASK)
123
124	/**
125	* Subset of the guest-CPU state that is shared between the guest and host.
126	*/
127	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
128
129	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
130	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
131	do { \
132	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
133	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
134	} while (0)
135
136	/** Assert that the required state bits are fetched. */
137	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
151
152	/** Assert that we're not executing a nested-guest. */
153	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
154	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
155	#else
156	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
157	#endif
158
159	/** Assert that we're executing a nested-guest. */
160	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
161	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
162	#else
163	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
164	#endif
165
166	/** Macro for checking and returning from the using function for
167	* \#VMEXIT intercepts that maybe caused during delivering of another
168	* event in the guest. */
169	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
170	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
171	do \
172	{ \
173	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
174	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
175	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
176	else if ( rc == VINF_EM_RESET \
177	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
178	{ \
179	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
180	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
181	} \
182	else \
183	return rc; \
184	} while (0)
185	#else
186	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
187	do \
188	{ \
189	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
190	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
191	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
192	else \
193	return rc; \
194	} while (0)
195	#endif
196
197	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
198	* instruction that exited. */
199	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
200	do { \
201	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
202	(a_rc) = VINF_EM_DBG_STEPPED; \
203	} while (0)
204
205	/** Validate segment descriptor granularity bit. */
206	#ifdef VBOX_STRICT
207	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
208	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
209	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
210	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
211	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
212	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
213	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
214	#else
215	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
216	#endif
217
218	/**
219	* Exception bitmap mask for all contributory exceptions.
220	*
221	* Page fault is deliberately excluded here as it's conditional as to whether
222	* it's contributory or benign. Page faults are handled separately.
223	*/
224	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
225	\| RT_BIT(X86_XCPT_DE))
226
227	/**
228	* Mandatory/unconditional guest control intercepts.
229	*
230	* SMIs can and do happen in normal operation. We need not intercept them
231	* while executing the guest (or nested-guest).
232	*/
233	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
234	\| SVM_CTRL_INTERCEPT_NMI \
235	\| SVM_CTRL_INTERCEPT_INIT \
236	\| SVM_CTRL_INTERCEPT_RDPMC \
237	\| SVM_CTRL_INTERCEPT_CPUID \
238	\| SVM_CTRL_INTERCEPT_RSM \
239	\| SVM_CTRL_INTERCEPT_HLT \
240	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
241	\| SVM_CTRL_INTERCEPT_MSR_PROT \
242	\| SVM_CTRL_INTERCEPT_INVLPGA \
243	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
244	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
245	\| SVM_CTRL_INTERCEPT_VMRUN \
246	\| SVM_CTRL_INTERCEPT_SKINIT \
247	\| SVM_CTRL_INTERCEPT_WBINVD \
248	\| SVM_CTRL_INTERCEPT_MONITOR \
249	\| SVM_CTRL_INTERCEPT_MWAIT \
250	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
251	\| SVM_CTRL_INTERCEPT_XSETBV)
252
253	/** @name VMCB Clean Bits.
254	*
255	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
256	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
257	* memory.
258	*
259	* @{ */
260	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
261	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
262	/** I/O permission bitmap, MSR permission bitmap. */
263	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
264	/** ASID. */
265	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
266	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
267	V_INTR_VECTOR. */
268	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
269	/** Nested Paging: Nested CR3 (nCR3), PAT. */
270	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
271	/** Control registers (CR0, CR3, CR4, EFER). */
272	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
273	/** Debug registers (DR6, DR7). */
274	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
275	/** GDT, IDT limit and base. */
276	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
277	/** Segment register: CS, SS, DS, ES limit and base. */
278	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
279	/** CR2.*/
280	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
281	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
282	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
283	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
284	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
285	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
286	/** Mask of all valid VMCB Clean bits. */
287	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
288	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
289	\| HMSVM_VMCB_CLEAN_ASID \
290	\| HMSVM_VMCB_CLEAN_INT_CTRL \
291	\| HMSVM_VMCB_CLEAN_NP \
292	\| HMSVM_VMCB_CLEAN_CRX_EFER \
293	\| HMSVM_VMCB_CLEAN_DRX \
294	\| HMSVM_VMCB_CLEAN_DT \
295	\| HMSVM_VMCB_CLEAN_SEG \
296	\| HMSVM_VMCB_CLEAN_CR2 \
297	\| HMSVM_VMCB_CLEAN_LBR \
298	\| HMSVM_VMCB_CLEAN_AVIC)
299	/** @} */
300
301	/**
302	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
303	*/
304	typedef enum SVMMSREXITREAD
305	{
306	/** Reading this MSR causes a \#VMEXIT. */
307	SVMMSREXIT_INTERCEPT_READ = 0xb,
308	/** Reading this MSR does not cause a \#VMEXIT. */
309	SVMMSREXIT_PASSTHRU_READ
310	} SVMMSREXITREAD;
311
312	/**
313	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
314	*/
315	typedef enum SVMMSREXITWRITE
316	{
317	/** Writing to this MSR causes a \#VMEXIT. */
318	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a \#VMEXIT. */
320	SVMMSREXIT_PASSTHRU_WRITE
321	} SVMMSREXITWRITE;
322
323	/**
324	* SVM \#VMEXIT handler.
325	*
326	* @returns Strict VBox status code.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pSvmTransient Pointer to the SVM-transient structure.
329	*/
330	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
331
332
333	/*********************************************************************************************************************************
334	* Internal Functions *
335	*********************************************************************************************************************************/
336	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
337	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
338
339
340	/** @name \#VMEXIT handlers.
341	* @{
342	*/
343	static FNSVMEXITHANDLER hmR0SvmExitIntr;
344	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
345	static FNSVMEXITHANDLER hmR0SvmExitInvd;
346	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
347	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
348	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
349	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
350	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
351	static FNSVMEXITHANDLER hmR0SvmExitHlt;
352	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
353	static FNSVMEXITHANDLER hmR0SvmExitMwait;
354	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
355	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
356	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
357	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
358	static FNSVMEXITHANDLER hmR0SvmExitMsr;
359	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
360	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
361	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
362	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
363	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
364	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
365	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
366	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
367	static FNSVMEXITHANDLER hmR0SvmExitPause;
368	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
369	static FNSVMEXITHANDLER hmR0SvmExitIret;
370	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
371	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
372	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
373	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
374	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
375	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
376	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
377	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
378	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
379	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
380	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
381	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
382	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
383	static FNSVMEXITHANDLER hmR0SvmExitClgi;
384	static FNSVMEXITHANDLER hmR0SvmExitStgi;
385	static FNSVMEXITHANDLER hmR0SvmExitVmload;
386	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
387	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
388	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
389	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
390	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
391	#endif
392	/** @} */
393
394	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
395	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
396	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
397	#endif
398	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
399
400
401	/*********************************************************************************************************************************
402	* Global Variables *
403	*********************************************************************************************************************************/
404	/** Ring-0 memory object for the IO bitmap. */
405	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
406	/** Physical address of the IO bitmap. */
407	static RTHCPHYS g_HCPhysIOBitmap;
408	/** Pointer to the IO bitmap. */
409	static R0PTRTYPE(void *) g_pvIOBitmap;
410
411	#ifdef VBOX_STRICT
412	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
413	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
414	# define HMSVM_LOG_CS RT_BIT_32(2)
415	# define HMSVM_LOG_SS RT_BIT_32(3)
416	# define HMSVM_LOG_FS RT_BIT_32(4)
417	# define HMSVM_LOG_GS RT_BIT_32(5)
418	# define HMSVM_LOG_LBR RT_BIT_32(6)
419	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
420	\| HMSVM_LOG_CR_REGS \
421	\| HMSVM_LOG_CS \
422	\| HMSVM_LOG_SS \
423	\| HMSVM_LOG_FS \
424	\| HMSVM_LOG_GS \
425	\| HMSVM_LOG_LBR)
426
427	/**
428	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
429	*
430	* @param pVCpu The cross context virtual CPU structure.
431	* @param pVmcb Pointer to the VM control block.
432	* @param pszPrefix Log prefix.
433	* @param fFlags Log flags, see HMSVM_LOG_XXX.
434	* @param uVerbose The verbosity level, currently unused.
435	*/
436	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
437	{
438	RT_NOREF2(pVCpu, uVerbose);
439	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
440
441	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
442	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
443
444	if (fFlags & HMSVM_LOG_RBP_RSP)
445	{
446	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
447	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
448	}
449
450	if (fFlags & HMSVM_LOG_CR_REGS)
451	{
452	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
453	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
454	}
455
456	if (fFlags & HMSVM_LOG_CS)
457	{
458	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
459	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
460	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
461	}
462	if (fFlags & HMSVM_LOG_SS)
463	{
464	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
465	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
466	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
467	}
468	if (fFlags & HMSVM_LOG_FS)
469	{
470	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
471	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
472	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
473	}
474	if (fFlags & HMSVM_LOG_GS)
475	{
476	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
477	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
478	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
479	}
480
481	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
482	if (fFlags & HMSVM_LOG_LBR)
483	{
484	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
485	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
486	}
487	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
488	}
489	#endif /* VBOX_STRICT */
490
491
492	/**
493	* Sets up and activates AMD-V on the current CPU.
494	*
495	* @returns VBox status code.
496	* @param pHostCpu The HM physical-CPU structure.
497	* @param pVM The cross context VM structure. Can be
498	* NULL after a resume!
499	* @param pvCpuPage Pointer to the global CPU page.
500	* @param HCPhysCpuPage Physical address of the global CPU page.
501	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
502	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
503	* unused).
504	*/
505	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
506	PCSUPHWVIRTMSRS pHwvirtMsrs)
507	{
508	Assert(!fEnabledByHost);
509	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
510	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
511	Assert(pvCpuPage); NOREF(pvCpuPage);
512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
513
514	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
515
516	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
517	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
518
519	/*
520	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
521	*/
522	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
523	if (u64HostEfer & MSR_K6_EFER_SVME)
524	{
525	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
526	if ( pVM
527	&& pVM->hm.s.svm.fIgnoreInUseError)
528	pHostCpu->fIgnoreAMDVInUseError = true;
529
530	if (!pHostCpu->fIgnoreAMDVInUseError)
531	{
532	ASMSetFlags(fEFlags);
533	return VERR_SVM_IN_USE;
534	}
535	}
536
537	/* Turn on AMD-V in the EFER MSR. */
538	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
539
540	/* Write the physical page address where the CPU will store the host state while executing the VM. */
541	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
542
543	/* Restore interrupts. */
544	ASMSetFlags(fEFlags);
545
546	/*
547	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
548	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
549	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
550	* entirely with before executing any guest code.
551	*/
552	pHostCpu->fFlushAsidBeforeUse = true;
553
554	/*
555	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
556	*/
557	++pHostCpu->cTlbFlushes;
558
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Deactivates AMD-V on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pHostCpu The HM physical-CPU structure.
568	* @param pvCpuPage Pointer to the global CPU page.
569	* @param HCPhysCpuPage Physical address of the global CPU page.
570	*/
571	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
572	{
573	RT_NOREF1(pHostCpu);
574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
575	AssertReturn( HCPhysCpuPage
576	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
577	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
578
579	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
580	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
581
582	/* Turn off AMD-V in the EFER MSR. */
583	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
584	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
585
586	/* Invalidate host state physical address. */
587	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
588
589	/* Restore interrupts. */
590	ASMSetFlags(fEFlags);
591
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Does global AMD-V initialization (called during module initialization).
598	*
599	* @returns VBox status code.
600	*/
601	VMMR0DECL(int) SVMR0GlobalInit(void)
602	{
603	/*
604	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
605	* intercept all IO accesses, it's done once globally here instead of per-VM.
606	*/
607	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
608	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
609	if (RT_FAILURE(rc))
610	return rc;
611
612	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
613	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
614
615	/* Set all bits to intercept all IO accesses. */
616	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
617
618	return VINF_SUCCESS;
619	}
620
621
622	/**
623	* Does global AMD-V termination (called during module termination).
624	*/
625	VMMR0DECL(void) SVMR0GlobalTerm(void)
626	{
627	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
628	{
629	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
630	g_pvIOBitmap = NULL;
631	g_HCPhysIOBitmap = 0;
632	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
633	}
634	}
635
636
637	/**
638	* Frees any allocated per-VCPU structures for a VM.
639	*
640	* @param pVM The cross context VM structure.
641	*/
642	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
643	{
644	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
645	{
646	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
647	AssertPtr(pVCpu);
648
649	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
650	{
651	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
652	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
653	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
654	}
655
656	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
657	{
658	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
659	pVCpu->hmr0.s.svm.pVmcb = NULL;
660	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
661	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
662	}
663
664	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
665	{
666	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
667	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
668	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
669	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
670	}
671	}
672	}
673
674
675	/**
676	* Sets pfnVMRun to the best suited variant.
677	*
678	* This must be called whenever anything changes relative to the SVMR0VMRun
679	* variant selection:
680	* - pVCpu->hm.s.fLoadSaveGuestXcr0
681	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
682	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
683	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
684	* - Perhaps: CPUMCTX.fXStateMask (windows only)
685	*
686	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
687	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
688	*/
689	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
690	{
691	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
692	{
693	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
694	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
695	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
696	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
697	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
698	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
699	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
700	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
701	};
702	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
703	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
704	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
705	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
706	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
707	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
708	}
709
710
711	/**
712	* Selector FNHMSVMVMRUN implementation.
713	*/
714	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
715	{
716	hmR0SvmUpdateVmRunFunction(pVCpu);
717	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
718	}
719
720
721	/**
722	* Does per-VM AMD-V initialization.
723	*
724	* @returns VBox status code.
725	* @param pVM The cross context VM structure.
726	*/
727	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
728	{
729	int rc = VERR_INTERNAL_ERROR_5;
730
731	/*
732	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
733	*/
734	uint32_t u32Family;
735	uint32_t u32Model;
736	uint32_t u32Stepping;
737	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
738	{
739	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
740	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
741	}
742
743	/*
744	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
745	*/
746	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
747	{
748	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
749	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
750	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
751	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
752	}
753
754	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
755	{
756	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
757
758	/*
759	* Initialize the hardware-assisted SVM guest-execution handler.
760	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
761	*/
762	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
763
764	/*
765	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
766	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
767	*/
768	/** @todo Does this need to be below 4G? */
769	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
770	if (RT_FAILURE(rc))
771	goto failure_cleanup;
772
773	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
774	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
775	Assert(pVCpu->hmr0.s.svm.HCPhysVmcbHost < _4G);
776	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
777
778	/*
779	* Allocate one page for the guest-state VMCB.
780	*/
781	/** @todo Does this need to be below 4G? */
782	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
783	if (RT_FAILURE(rc))
784	goto failure_cleanup;
785
786	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
787	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
788	Assert(pVCpu->hmr0.s.svm.HCPhysVmcb < _4G);
789	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
790
791	/*
792	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
793	* SVM to not require one.
794	*/
795	/** @todo Does this need to be below 4G? */
796	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
797	false /* fExecutable */);
798	if (RT_FAILURE(rc))
799	goto failure_cleanup;
800
801	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
802	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
803	/* Set all bits to intercept all MSR accesses (changed later on). */
804	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
805	}
806
807	return VINF_SUCCESS;
808
809	failure_cleanup:
810	hmR0SvmFreeStructs(pVM);
811	return rc;
812	}
813
814
815	/**
816	* Does per-VM AMD-V termination.
817	*
818	* @returns VBox status code.
819	* @param pVM The cross context VM structure.
820	*/
821	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
822	{
823	hmR0SvmFreeStructs(pVM);
824	return VINF_SUCCESS;
825	}
826
827
828	/**
829	* Returns whether the VMCB Clean Bits feature is supported.
830	*
831	* @returns @c true if supported, @c false otherwise.
832	* @param pVCpu The cross context virtual CPU structure.
833	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
834	*/
835	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
836	{
837	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
838	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
839	if (!fIsNestedGuest)
840	return fHostVmcbCleanBits;
841	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
842	}
843
844
845	/**
846	* Returns whether the decode assists feature is supported.
847	*
848	* @returns @c true if supported, @c false otherwise.
849	* @param pVCpu The cross context virtual CPU structure.
850	*/
851	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
852	{
853	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
854	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
855	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
856	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
857	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
858	#endif
859	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
860	}
861
862
863	/**
864	* Returns whether the NRIP_SAVE feature is supported.
865	*
866	* @returns @c true if supported, @c false otherwise.
867	* @param pVCpu The cross context virtual CPU structure.
868	*/
869	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
870	{
871	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
872	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
873	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
874	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
875	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
876	#endif
877	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
878	}
879
880
881	/**
882	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
883	*
884	* @param pVCpu The cross context virtual CPU structure.
885	* @param pbMsrBitmap Pointer to the MSR bitmap.
886	* @param idMsr The MSR for which the permissions are being set.
887	* @param enmRead MSR read permissions.
888	* @param enmWrite MSR write permissions.
889	*
890	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
891	* caller needs to take care of this.
892	*/
893	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
894	SVMMSREXITWRITE enmWrite)
895	{
896	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
897	uint16_t offMsrpm;
898	uint8_t uMsrpmBit;
899	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
900	AssertRC(rc);
901
902	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
903	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
904
905	pbMsrBitmap += offMsrpm;
906	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
907	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
908	else
909	{
910	if (!fInNestedGuestMode)
911	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
912	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
913	else
914	{
915	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
916	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
917	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
918	else
919	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
920	}
921	#endif
922	}
923
924	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
925	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
926	else
927	{
928	if (!fInNestedGuestMode)
929	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
930	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
931	else
932	{
933	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
934	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
935	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
936	else
937	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
938	}
939	#endif
940	}
941	}
942
943
944	/**
945	* Sets up AMD-V for the specified VM.
946	* This function is only called once per-VM during initalization.
947	*
948	* @returns VBox status code.
949	* @param pVM The cross context VM structure.
950	*/
951	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
952	{
953	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
954	AssertReturn(pVM, VERR_INVALID_PARAMETER);
955
956	/*
957	* Validate and copy over some parameters.
958	*/
959	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
960	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
961	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
962	pVM->hmr0.s.fNestedPaging = fNestedPaging;
963	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
964
965	/*
966	* Determin some configuration parameters.
967	*/
968	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
969	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
970	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
971
972	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
973	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
974
975	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
976	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
977	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
978
979	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
980	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
981	#endif
982
983	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
984	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
985	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
986	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
987
988	/* Always trap #AC for reasons of security. */
989	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
990
991	/* Always trap #DB for reasons of security. */
992	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
993
994	/* Trap exceptions unconditionally (debug purposes). */
995	#ifdef HMSVM_ALWAYS_TRAP_PF
996	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
997	#endif
998	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
999	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
1000	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
1001	\| RT_BIT_32(X86_XCPT_DE)
1002	\| RT_BIT_32(X86_XCPT_NM)
1003	\| RT_BIT_32(X86_XCPT_UD)
1004	\| RT_BIT_32(X86_XCPT_NP)
1005	\| RT_BIT_32(X86_XCPT_SS)
1006	\| RT_BIT_32(X86_XCPT_GP)
1007	\| RT_BIT_32(X86_XCPT_PF)
1008	\| RT_BIT_32(X86_XCPT_MF)
1009	;
1010	#endif
1011
1012	/* Apply the exceptions intercepts needed by the GIM provider. */
1013	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1014	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1015
1016	/* Apply the exceptions intercepts needed by the GCM fixers. */
1017	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1018	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1019
1020	/* The mesa 3d driver hack needs #GP. */
1021	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1022	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1023
1024	/* Set up unconditional intercepts and conditions. */
1025	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1026	\| SVM_CTRL_INTERCEPT_VMMCALL
1027	\| SVM_CTRL_INTERCEPT_VMSAVE
1028	\| SVM_CTRL_INTERCEPT_VMLOAD
1029	\| SVM_CTRL_INTERCEPT_CLGI
1030	\| SVM_CTRL_INTERCEPT_STGI;
1031
1032	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1033	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1034	#endif
1035
1036	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1037	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1038	{
1039	/* Virtualized VMSAVE/VMLOAD. */
1040	if (fUseVirtVmsaveVmload)
1041	{
1042	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1043	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1044	\| SVM_CTRL_INTERCEPT_VMLOAD);
1045	}
1046	else
1047	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1048
1049	/* Virtual GIF. */
1050	if (fUseVGif)
1051	{
1052	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1053	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1054	\| SVM_CTRL_INTERCEPT_STGI);
1055	}
1056	else
1057	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1058	}
1059	else
1060	#endif
1061	{
1062	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1063	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1064	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1065	}
1066
1067	/* CR4 writes must always be intercepted for tracking PGM mode changes and
1068	AVX (for XCR0 syncing during worlds switching). */
1069	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1070
1071	/* Intercept all DRx reads and writes by default. Changed later on. */
1072	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1073	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1074
1075	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1076	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1077
1078	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1079	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1080	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1081
1082	/* Set the IO permission bitmap physical addresses. */
1083	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1084
1085	/* LBR virtualization. */
1086	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1087
1088	/* The host ASID MBZ, for the guest start with 1. */
1089	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1090
1091	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1092	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1093
1094	/* Without Nested Paging, we need additionally intercepts. */
1095	if (!fNestedPaging)
1096	{
1097	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1098	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1099	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1100
1101	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1102	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1103	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1104
1105	/* Page faults must be intercepted to implement shadow paging. */
1106	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1107	}
1108
1109	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1110	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1111	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1112
1113	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1114	if (fUsePauseFilter)
1115	{
1116	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1117	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1118	if (fPauseFilterThreshold)
1119	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1120	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1121	}
1122
1123	/*
1124	* Setup the MSR permission bitmap.
1125	* The following MSRs are saved/restored automatically during the world-switch.
1126	* Don't intercept guest read/write accesses to these MSRs.
1127	*/
1128	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1129	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1130	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1131	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1132	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1133	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1134	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1135	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1136	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1137	{
1138	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1139	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1140	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1141	}
1142	else
1143	{
1144	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1145	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1146	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1147	}
1148	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1149
1150	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1151	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1152
1153	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1154	{
1155	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1156	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1157	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1158	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1159
1160	/* Copy the VMCB control area. */
1161	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1162
1163	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1164	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1165	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1166	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1167
1168	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1169	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1170
1171	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1172	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1173	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1174	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1175	}
1176
1177	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1178	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1179	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1180	#else
1181	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1182	#endif
1183	return VINF_SUCCESS;
1184	}
1185
1186
1187	/**
1188	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1189	*
1190	* @returns Pointer to the current context VMCB.
1191	* @param pVCpu The cross context virtual CPU structure.
1192	*/
1193	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1194	{
1195	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1196	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1197	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1198	#endif
1199	return pVCpu->hmr0.s.svm.pVmcb;
1200	}
1201
1202
1203	/**
1204	* Gets a pointer to the nested-guest VMCB cache.
1205	*
1206	* @returns Pointer to the nested-guest VMCB cache.
1207	* @param pVCpu The cross context virtual CPU structure.
1208	*/
1209	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1210	{
1211	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1212	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1213	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1214	#else
1215	RT_NOREF(pVCpu);
1216	return NULL;
1217	#endif
1218	}
1219
1220
1221	/**
1222	* Invalidates a guest page by guest virtual address.
1223	*
1224	* @returns VBox status code.
1225	* @param pVCpu The cross context virtual CPU structure.
1226	* @param GCVirt Guest virtual address of the page to invalidate.
1227	*/
1228	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1229	{
1230	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1231
1232	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1233
1234	/* Skip it if a TLB flush is already pending. */
1235	if (!fFlushPending)
1236	{
1237	Log4Func(("%#RGv\n", GCVirt));
1238
1239	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1240	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1241
1242	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1243	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1244	}
1245	return VINF_SUCCESS;
1246	}
1247
1248
1249	/**
1250	* Flushes the appropriate tagged-TLB entries.
1251	*
1252	* @param pHostCpu The HM physical-CPU structure.
1253	* @param pVCpu The cross context virtual CPU structure.
1254	* @param pVmcb Pointer to the VM control block.
1255	*/
1256	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1257	{
1258	/*
1259	* Force a TLB flush for the first world switch if the current CPU differs from the one
1260	* we ran on last. This can happen both for start & resume due to long jumps back to
1261	* ring-3.
1262	*
1263	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1264	* correlation between it and the physical CPU.
1265	*
1266	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1267	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1268	*/
1269	bool fNewAsid = false;
1270	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1271	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1272	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1273	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1274	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1275	#endif
1276	)
1277	{
1278	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1279	pVCpu->hmr0.s.fForceTLBFlush = true;
1280	fNewAsid = true;
1281	}
1282
1283	/* Set TLB flush state as checked until we return from the world switch. */
1284	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1285
1286	/* Check for explicit TLB flushes. */
1287	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1288	{
1289	pVCpu->hmr0.s.fForceTLBFlush = true;
1290	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1291	}
1292
1293	/*
1294	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1295	* This Host CPU requirement takes precedence.
1296	*/
1297	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1298	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1299	{
1300	pHostCpu->uCurrentAsid = 1;
1301	pVCpu->hmr0.s.uCurrentAsid = 1;
1302	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1303	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1304	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1305
1306	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1307	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1308	}
1309	else
1310	{
1311	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1312	if (pVCpu->hmr0.s.fForceTLBFlush)
1313	{
1314	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1315	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1316
1317	if (fNewAsid)
1318	{
1319	++pHostCpu->uCurrentAsid;
1320
1321	bool fHitASIDLimit = false;
1322	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1323	{
1324	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1325	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1326	fHitASIDLimit = true;
1327	}
1328
1329	if ( fHitASIDLimit
1330	\|\| pHostCpu->fFlushAsidBeforeUse)
1331	{
1332	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1333	pHostCpu->fFlushAsidBeforeUse = false;
1334	}
1335
1336	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1337	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1338	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1339	}
1340	else
1341	{
1342	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1343	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1344	else
1345	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1346	}
1347
1348	pVCpu->hmr0.s.fForceTLBFlush = false;
1349	}
1350	}
1351
1352	/* Update VMCB with the ASID. */
1353	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1354	{
1355	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1356	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1357	}
1358
1359	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1360	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1361	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1362	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1363	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1364	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1365	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1366	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1367
1368	#ifdef VBOX_WITH_STATISTICS
1369	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1370	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1371	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1372	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1373	{
1374	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1375	}
1376	else
1377	{
1378	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1379	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1380	}
1381	#endif
1382	}
1383
1384
1385	/**
1386	* Sets an exception intercept in the specified VMCB.
1387	*
1388	* @param pVmcb Pointer to the VM control block.
1389	* @param uXcpt The exception (X86_XCPT_*).
1390	*/
1391	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1392	{
1393	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1394	{
1395	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1396	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1397	}
1398	}
1399
1400
1401	/**
1402	* Clears an exception intercept in the specified VMCB.
1403	*
1404	* @param pVCpu The cross context virtual CPU structure.
1405	* @param pVmcb Pointer to the VM control block.
1406	* @param uXcpt The exception (X86_XCPT_*).
1407	*
1408	* @remarks This takes into account if we're executing a nested-guest and only
1409	* removes the exception intercept if both the guest -and- nested-guest
1410	* are not intercepting it.
1411	*/
1412	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1413	{
1414	Assert(uXcpt != X86_XCPT_DB);
1415	Assert(uXcpt != X86_XCPT_AC);
1416	Assert(uXcpt != X86_XCPT_GP);
1417	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1418	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1419	{
1420	bool fRemove = true;
1421	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1422	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1423	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1424	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1425	{
1426	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1427	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1428	}
1429	# else
1430	RT_NOREF(pVCpu);
1431	# endif
1432	if (fRemove)
1433	{
1434	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1435	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1436	}
1437	}
1438	#else
1439	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1440	#endif
1441	}
1442
1443
1444	/**
1445	* Sets a control intercept in the specified VMCB.
1446	*
1447	* @param pVmcb Pointer to the VM control block.
1448	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1449	*/
1450	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1451	{
1452	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1453	{
1454	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1455	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1456	}
1457	}
1458
1459
1460	/**
1461	* Clears a control intercept in the specified VMCB.
1462	*
1463	* @returns @c true if the intercept is still set, @c false otherwise.
1464	* @param pVCpu The cross context virtual CPU structure.
1465	* @param pVmcb Pointer to the VM control block.
1466	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1467	*
1468	* @remarks This takes into account if we're executing a nested-guest and only
1469	* removes the control intercept if both the guest -and- nested-guest
1470	* are not intercepting it.
1471	*/
1472	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1473	{
1474	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1475	{
1476	bool fRemove = true;
1477	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1478	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1479	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1480	{
1481	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1482	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1483	}
1484	#else
1485	RT_NOREF(pVCpu);
1486	#endif
1487	if (fRemove)
1488	{
1489	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1490	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1491	}
1492	}
1493
1494	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1495	}
1496
1497
1498	/**
1499	* Exports the guest (or nested-guest) CR0 into the VMCB.
1500	*
1501	* @param pVCpu The cross context virtual CPU structure.
1502	* @param pVmcb Pointer to the VM control block.
1503	*
1504	* @remarks This assumes we always pre-load the guest FPU.
1505	* @remarks No-long-jump zone!!!
1506	*/
1507	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1508	{
1509	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1510
1511	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1512	uint64_t const uGuestCr0 = pCtx->cr0;
1513	uint64_t uShadowCr0 = uGuestCr0;
1514
1515	/* Always enable caching. */
1516	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1517
1518	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1519	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1520	{
1521	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1522	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1523	}
1524
1525	/*
1526	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1527	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1528	* see @bugref{7243#c103}.
1529	*/
1530	if (!(uGuestCr0 & X86_CR0_NE))
1531	{
1532	uShadowCr0 \|= X86_CR0_NE;
1533	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1534	}
1535	else
1536	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1537
1538	/*
1539	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1540	*
1541	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1542	* see @bugref{6944}.
1543	*
1544	* We also don't ever want to honor weird things like cache disable from the guest.
1545	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1546	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1547	*/
1548	if (uShadowCr0 == uGuestCr0)
1549	{
1550	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1551	{
1552	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1553	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1554	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1555	}
1556	else
1557	{
1558	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1559	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1560	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1561	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1562	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1563	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1564	}
1565	}
1566	else
1567	{
1568	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1569	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1570	}
1571	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1572
1573	Assert(!RT_HI_U32(uShadowCr0));
1574	if (pVmcb->guest.u64CR0 != uShadowCr0)
1575	{
1576	pVmcb->guest.u64CR0 = uShadowCr0;
1577	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1578	}
1579	}
1580
1581
1582	/**
1583	* Exports the guest (or nested-guest) CR3 into the VMCB.
1584	*
1585	* @param pVCpu The cross context virtual CPU structure.
1586	* @param pVmcb Pointer to the VM control block.
1587	*
1588	* @remarks No-long-jump zone!!!
1589	*/
1590	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1591	{
1592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593
1594	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1595	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1596	if (pVM->hmr0.s.fNestedPaging)
1597	{
1598	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1599	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1600	pVmcb->guest.u64CR3 = pCtx->cr3;
1601	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1602	}
1603	else
1604	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1605
1606	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1607	}
1608
1609
1610	/**
1611	* Exports the guest (or nested-guest) CR4 into the VMCB.
1612	*
1613	* @param pVCpu The cross context virtual CPU structure.
1614	* @param pVmcb Pointer to the VM control block.
1615	*
1616	* @remarks No-long-jump zone!!!
1617	*/
1618	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1619	{
1620	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1621
1622	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1623	uint64_t uShadowCr4 = pCtx->cr4;
1624	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1625	{
1626	switch (pVCpu->hm.s.enmShadowMode)
1627	{
1628	case PGMMODE_REAL:
1629	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1630	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1631
1632	case PGMMODE_32_BIT: /* 32-bit paging. */
1633	uShadowCr4 &= ~X86_CR4_PAE;
1634	break;
1635
1636	case PGMMODE_PAE: /* PAE paging. */
1637	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1638	/** Must use PAE paging as we could use physical memory > 4 GB */
1639	uShadowCr4 \|= X86_CR4_PAE;
1640	break;
1641
1642	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1643	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1644	#ifdef VBOX_WITH_64_BITS_GUESTS
1645	break;
1646	#else
1647	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1648	#endif
1649
1650	default: /* shut up gcc */
1651	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1652	}
1653	}
1654
1655	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1656	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1657	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1658	{
1659	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1660	hmR0SvmUpdateVmRunFunction(pVCpu);
1661	}
1662
1663	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1664	if (uShadowCr4 == pCtx->cr4)
1665	{
1666	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1667	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1668	else
1669	{
1670	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1671	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1672	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1673	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1674	}
1675	}
1676	else
1677	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1678
1679	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking
1680	PGM mode changes and AVX (for XCR0 syncing during worlds switching). */
1681	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1682
1683	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1684	Assert(!RT_HI_U32(uShadowCr4));
1685	pVmcb->guest.u64CR4 = uShadowCr4;
1686	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1687
1688	return VINF_SUCCESS;
1689	}
1690
1691
1692	/**
1693	* Exports the guest (or nested-guest) control registers into the VMCB.
1694	*
1695	* @returns VBox status code.
1696	* @param pVCpu The cross context virtual CPU structure.
1697	* @param pVmcb Pointer to the VM control block.
1698	*
1699	* @remarks No-long-jump zone!!!
1700	*/
1701	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1702	{
1703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1704
1705	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1706	{
1707	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1708	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1709
1710	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1711	{
1712	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1713	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1714	}
1715
1716	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1717	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1718
1719	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1720	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1721	{
1722	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1723	if (RT_FAILURE(rc))
1724	return rc;
1725	}
1726
1727	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1728	}
1729	return VINF_SUCCESS;
1730	}
1731
1732
1733	/**
1734	* Exports the guest (or nested-guest) segment registers into the VMCB.
1735	*
1736	* @returns VBox status code.
1737	* @param pVCpu The cross context virtual CPU structure.
1738	* @param pVmcb Pointer to the VM control block.
1739	*
1740	* @remarks No-long-jump zone!!!
1741	*/
1742	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1743	{
1744	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1745	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1746
1747	/* Guest segment registers. */
1748	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1749	{
1750	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1751	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1752
1753	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1754	{
1755	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1756	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1757	}
1758
1759	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1760	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1761
1762	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1763	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1764
1765	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1766	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1767
1768	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1769	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1770
1771	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1772	}
1773
1774	/* Guest TR. */
1775	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1776	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1777
1778	/* Guest LDTR. */
1779	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1780	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1781
1782	/* Guest GDTR. */
1783	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1784	{
1785	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1786	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1787	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1788	}
1789
1790	/* Guest IDTR. */
1791	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1792	{
1793	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1794	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1795	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1796	}
1797
1798	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1799	\| HM_CHANGED_GUEST_TABLE_MASK);
1800	}
1801
1802
1803	/**
1804	* Exports the guest (or nested-guest) MSRs into the VMCB.
1805	*
1806	* @param pVCpu The cross context virtual CPU structure.
1807	* @param pVmcb Pointer to the VM control block.
1808	*
1809	* @remarks No-long-jump zone!!!
1810	*/
1811	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1812	{
1813	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1814	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1815
1816	/* Guest Sysenter MSRs. */
1817	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1818	{
1819	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1820	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1821
1822	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1823	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1824
1825	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1826	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1827	}
1828
1829	/*
1830	* Guest EFER MSR.
1831	* AMD-V requires guest EFER.SVME to be set. Weird.
1832	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1833	*/
1834	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1835	{
1836	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1837	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1838	}
1839
1840	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1841	if ( !CPUMIsGuestInLongModeEx(pCtx)
1842	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1843	{
1844	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1845	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1846	}
1847
1848	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1849	{
1850	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1851	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1852	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1853	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1854	}
1855
1856	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1857	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1858
1859	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1860	\| HM_CHANGED_GUEST_EFER_MSR
1861	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1862	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1863
1864	/*
1865	* Setup the PAT MSR (applicable for Nested Paging only).
1866	*
1867	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1868	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1869	*
1870	* While guests can modify and see the modified values through the shadow values,
1871	* we shall not honor any guest modifications of this MSR to ensure caching is always
1872	* enabled similar to how we clear CR0.CD and NW bits.
1873	*
1874	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1875	*/
1876	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1877
1878	/* Enable the last branch record bit if LBR virtualization is enabled. */
1879	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1880	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1881	}
1882
1883
1884	/**
1885	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1886	* the necessary intercepts accordingly.
1887	*
1888	* @param pVCpu The cross context virtual CPU structure.
1889	* @param pVmcb Pointer to the VM control block.
1890	*
1891	* @remarks No-long-jump zone!!!
1892	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1893	*/
1894	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1895	{
1896	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1897
1898	/** @todo Figure out stepping with nested-guest. */
1899	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1900	{
1901	/*
1902	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1903	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1904	* Instead, they are strictly only requested when the nested hypervisor intercepts
1905	* them -- handled while merging VMCB controls.
1906	*
1907	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1908	* then the nested-guest debug state should be actively loaded on the host so that
1909	* nested-guest reads/writes its own debug registers without causing VM-exits.
1910	*/
1911	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1912	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1913	&& !CPUMIsGuestDebugStateActive(pVCpu))
1914	{
1915	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1916	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1917	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1918	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1919	}
1920
1921	pVmcb->guest.u64DR6 = pCtx->dr[6];
1922	pVmcb->guest.u64DR7 = pCtx->dr[7];
1923	return;
1924	}
1925
1926	/*
1927	* Anyone single stepping on the host side? If so, we'll have to use the
1928	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1929	* the VMM level like the VT-x implementations does.
1930	*/
1931	bool fInterceptMovDRx = false;
1932	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1933	if (fStepping)
1934	{
1935	pVCpu->hmr0.s.fClearTrapFlag = true;
1936	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1937	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1938	}
1939
1940	if ( fStepping
1941	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1942	{
1943	/*
1944	* Use the combined guest and host DRx values found in the hypervisor
1945	* register set because the debugger has breakpoints active or someone
1946	* is single stepping on the host side.
1947	*
1948	* Note! DBGF expects a clean DR6 state before executing guest code.
1949	*/
1950	if (!CPUMIsHyperDebugStateActive(pVCpu))
1951	{
1952	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1953	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1954	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1955	}
1956
1957	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1958	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1959	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1960	{
1961	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1962	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1963	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1964	}
1965
1966	/** @todo If we cared, we could optimize to allow the guest to read registers
1967	* with the same values. */
1968	fInterceptMovDRx = true;
1969	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1970	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1971	}
1972	else
1973	{
1974	/*
1975	* Update DR6, DR7 with the guest values if necessary.
1976	*/
1977	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1978	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1979	{
1980	pVmcb->guest.u64DR7 = pCtx->dr[7];
1981	pVmcb->guest.u64DR6 = pCtx->dr[6];
1982	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1983	}
1984	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1985
1986	/*
1987	* If the guest has enabled debug registers, we need to load them prior to
1988	* executing guest code so they'll trigger at the right time.
1989	*/
1990	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1991	{
1992	if (!CPUMIsGuestDebugStateActive(pVCpu))
1993	{
1994	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1995	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1996	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1997	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1998	}
1999	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2000	}
2001	/*
2002	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2003	* intercept #DB as DR6 is updated in the VMCB.
2004	*
2005	* Note! If we cared and dared, we could skip intercepting \#DB here.
2006	* However, \#DB shouldn't be performance critical, so we'll play safe
2007	* and keep the code similar to the VT-x code and always intercept it.
2008	*/
2009	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2010	fInterceptMovDRx = true;
2011	}
2012
2013	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2014	if (fInterceptMovDRx)
2015	{
2016	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2017	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2018	{
2019	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2020	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2021	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2022	}
2023	}
2024	else
2025	{
2026	if ( pVmcb->ctrl.u16InterceptRdDRx
2027	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2028	{
2029	pVmcb->ctrl.u16InterceptRdDRx = 0;
2030	pVmcb->ctrl.u16InterceptWrDRx = 0;
2031	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2032	}
2033	}
2034	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2035	}
2036
2037	/**
2038	* Exports the hardware virtualization state into the nested-guest
2039	* VMCB.
2040	*
2041	* @param pVCpu The cross context virtual CPU structure.
2042	* @param pVmcb Pointer to the VM control block.
2043	*
2044	* @remarks No-long-jump zone!!!
2045	*/
2046	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2047	{
2048	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2049
2050	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2051	{
2052	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2053	{
2054	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2055	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2056
2057	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2058	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2059	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2060	NOREF(pVM);
2061
2062	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2063	}
2064
2065	/*
2066	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2067	* since SVM doesn't have a preemption timer.
2068	*
2069	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2070	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2071	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2072	*/
2073	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2074	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2075	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2076	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2077	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2078	{
2079	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2080	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2081	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2082	}
2083	else
2084	{
2085	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2086	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2087	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2088	}
2089	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2090
2091	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2092	}
2093	}
2094
2095
2096	/**
2097	* Exports the guest APIC TPR state into the VMCB.
2098	*
2099	* @returns VBox status code.
2100	* @param pVCpu The cross context virtual CPU structure.
2101	* @param pVmcb Pointer to the VM control block.
2102	*/
2103	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2104	{
2105	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2106
2107	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2108	{
2109	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2110	if ( PDMHasApic(pVM)
2111	&& APICIsEnabled(pVCpu))
2112	{
2113	bool fPendingIntr;
2114	uint8_t u8Tpr;
2115	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2116	AssertRCReturn(rc, rc);
2117
2118	/* Assume that we need to trap all TPR accesses and thus need not check on
2119	every #VMEXIT if we should update the TPR. */
2120	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2121	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2122
2123	if (!pVM->hm.s.fTprPatchingActive)
2124	{
2125	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2126	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2127
2128	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2129	can deliver the interrupt to the guest. */
2130	if (fPendingIntr)
2131	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2132	else
2133	{
2134	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2135	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2136	}
2137
2138	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2139	}
2140	else
2141	{
2142	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2143	pVmcb->guest.u64LSTAR = u8Tpr;
2144	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2145
2146	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2147	if (fPendingIntr)
2148	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2149	else
2150	{
2151	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2152	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2153	}
2154	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2155	}
2156	}
2157	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2158	}
2159	return VINF_SUCCESS;
2160	}
2161
2162
2163	/**
2164	* Sets up the exception interrupts required for guest execution in the VMCB.
2165	*
2166	* @param pVCpu The cross context virtual CPU structure.
2167	* @param pVmcb Pointer to the VM control block.
2168	*
2169	* @remarks No-long-jump zone!!!
2170	*/
2171	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2172	{
2173	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2174
2175	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2176	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2177	{
2178	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2179	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2180	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2181	else
2182	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2183
2184	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2185	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2186	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2187	else
2188	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2189
2190	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2191	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2192	}
2193	}
2194
2195
2196	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2197	/**
2198	* Merges guest and nested-guest intercepts for executing the nested-guest using
2199	* hardware-assisted SVM.
2200	*
2201	* This merges the guest and nested-guest intercepts in a way that if the outer
2202	* guest intercept is set we need to intercept it in the nested-guest as
2203	* well.
2204	*
2205	* @param pVCpu The cross context virtual CPU structure.
2206	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2207	*/
2208	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2209	{
2210	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2211	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2212	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2213	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2214
2215	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2216	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2217	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2218
2219	/* Always intercept CR4 writes for tracking PGM mode changes and AVX (for
2220	XCR0 syncing during worlds switching). */
2221	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2222
2223	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2224	if (!pVM->hmr0.s.fNestedPaging)
2225	{
2226	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2227	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2228	}
2229
2230	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2231	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2232	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2233
2234	/*
2235	* Merge the guest's exception intercepts into the nested-guest VMCB.
2236	*
2237	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2238	* while executing the nested-guest.
2239	*
2240	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2241	* be tweaked later depending on how we wish to implement breakpoints.
2242	*
2243	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2244	* loaded into their guests, not ours.
2245	*
2246	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2247	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2248	*/
2249	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2250	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2251	& ~( RT_BIT(X86_XCPT_UD)
2252	\| RT_BIT(X86_XCPT_BP)
2253	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2254	#else
2255	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2256	#endif
2257
2258	/*
2259	* Adjust intercepts while executing the nested-guest that differ from the
2260	* outer guest intercepts.
2261	*
2262	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2263	* that belong to the nested-guest to the outer guest.
2264	*
2265	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2266	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2267	*/
2268	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2269	\| SVM_CTRL_INTERCEPT_VMMCALL))
2270	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2271
2272	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2273	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2274
2275	/* Finally, update the VMCB clean bits. */
2276	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2277	}
2278	#endif
2279
2280
2281	/**
2282	* Enters the AMD-V session.
2283	*
2284	* @returns VBox status code.
2285	* @param pVCpu The cross context virtual CPU structure.
2286	*/
2287	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2288	{
2289	AssertPtr(pVCpu);
2290	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2291	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2292
2293	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2294	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2295	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2296
2297	pVCpu->hmr0.s.fLeaveDone = false;
2298	return VINF_SUCCESS;
2299	}
2300
2301
2302	/**
2303	* Thread-context callback for AMD-V.
2304	*
2305	* This is used together with RTThreadCtxHookCreate() on platforms which
2306	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2307	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2308	*
2309	* @param enmEvent The thread-context event.
2310	* @param pVCpu The cross context virtual CPU structure.
2311	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2312	* @thread EMT(pVCpu)
2313	*/
2314	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2315	{
2316	NOREF(fGlobalInit);
2317
2318	switch (enmEvent)
2319	{
2320	case RTTHREADCTXEVENT_OUT:
2321	{
2322	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2323	VMCPU_ASSERT_EMT(pVCpu);
2324
2325	/* No longjmps (log-flush, locks) in this fragile context. */
2326	VMMRZCallRing3Disable(pVCpu);
2327
2328	if (!pVCpu->hmr0.s.fLeaveDone)
2329	{
2330	hmR0SvmLeave(pVCpu, false /* fImportState */);
2331	pVCpu->hmr0.s.fLeaveDone = true;
2332	}
2333
2334	/* Leave HM context, takes care of local init (term). */
2335	int rc = HMR0LeaveCpu(pVCpu);
2336	AssertRC(rc); NOREF(rc);
2337
2338	/* Restore longjmp state. */
2339	VMMRZCallRing3Enable(pVCpu);
2340	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2341	break;
2342	}
2343
2344	case RTTHREADCTXEVENT_IN:
2345	{
2346	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2347	VMCPU_ASSERT_EMT(pVCpu);
2348
2349	/* No longjmps (log-flush, locks) in this fragile context. */
2350	VMMRZCallRing3Disable(pVCpu);
2351
2352	/*
2353	* Initialize the bare minimum state required for HM. This takes care of
2354	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2355	*/
2356	int rc = hmR0EnterCpu(pVCpu);
2357	AssertRC(rc); NOREF(rc);
2358	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2359	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2360
2361	pVCpu->hmr0.s.fLeaveDone = false;
2362
2363	/* Restore longjmp state. */
2364	VMMRZCallRing3Enable(pVCpu);
2365	break;
2366	}
2367
2368	default:
2369	break;
2370	}
2371	}
2372
2373
2374	/**
2375	* Saves the host state.
2376	*
2377	* @returns VBox status code.
2378	* @param pVCpu The cross context virtual CPU structure.
2379	*
2380	* @remarks No-long-jump zone!!!
2381	*/
2382	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2383	{
2384	NOREF(pVCpu);
2385
2386	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2387	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2388	return VINF_SUCCESS;
2389	}
2390
2391
2392	/**
2393	* Exports the guest or nested-guest state from the virtual-CPU context into the
2394	* VMCB.
2395	*
2396	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2397	* code based on the virtual-CPU mode.
2398	*
2399	* @returns VBox status code.
2400	* @param pVCpu The cross context virtual CPU structure.
2401	* @param pSvmTransient Pointer to the SVM-transient structure.
2402	*
2403	* @remarks No-long-jump zone!!!
2404	*/
2405	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2406	{
2407	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2408
2409	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2410	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2411	Assert(pVmcb);
2412
2413	pVmcb->guest.u64RIP = pCtx->rip;
2414	pVmcb->guest.u64RSP = pCtx->rsp;
2415	pVmcb->guest.u64RFlags = pCtx->eflags.u;
2416	pVmcb->guest.u64RAX = pCtx->rax;
2417
2418	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2419	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2420
2421	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2422	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2423	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2424	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2425	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2426
2427	ASMSetFlags(fEFlags);
2428
2429	if (!fIsNestedGuest)
2430	{
2431	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2432	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2433	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2434	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2435	}
2436
2437	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2438	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2439	\| HM_CHANGED_GUEST_RFLAGS
2440	\| HM_CHANGED_GUEST_GPRS_MASK
2441	\| HM_CHANGED_GUEST_X87
2442	\| HM_CHANGED_GUEST_SSE_AVX
2443	\| HM_CHANGED_GUEST_OTHER_XSAVE
2444	\| HM_CHANGED_GUEST_XCRx
2445	\| HM_CHANGED_GUEST_TSC_AUX
2446	\| HM_CHANGED_GUEST_OTHER_MSRS;
2447	if (fIsNestedGuest)
2448	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2449	\| HM_CHANGED_GUEST_APIC_TPR;
2450
2451	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2452	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2453
2454	#ifdef VBOX_STRICT
2455	/*
2456	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2457	* except for the host-context and/or shared host-guest context bits.
2458	*/
2459	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2460	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2461	("fCtxChanged=%#RX64\n", fCtxChanged));
2462
2463	/*
2464	* If we need to log state that isn't always imported, we'll need to import them here.
2465	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2466	*/
2467	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2468	#endif
2469
2470	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2471	return VINF_SUCCESS;
2472	}
2473
2474	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2475
2476	/**
2477	* Merges the guest and nested-guest MSR permission bitmap.
2478	*
2479	* If the guest is intercepting an MSR we need to intercept it regardless of
2480	* whether the nested-guest is intercepting it or not.
2481	*
2482	* @param pHostCpu The HM physical-CPU structure.
2483	* @param pVCpu The cross context virtual CPU structure.
2484	*
2485	* @remarks No-long-jmp zone!!!
2486	*/
2487	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2488	{
2489	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2490	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2491	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2492
2493	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2494	uint32_t const offRsvdQwords = 0x1800 >> 3;
2495	for (uint32_t i = 0; i < offRsvdQwords; i++)
2496	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2497	}
2498
2499
2500	/**
2501	* Caches the nested-guest VMCB fields before we modify them for execution using
2502	* hardware-assisted SVM.
2503	*
2504	* @returns true if the VMCB was previously already cached, false otherwise.
2505	* @param pVCpu The cross context virtual CPU structure.
2506	*
2507	* @sa HMNotifySvmNstGstVmexit.
2508	*/
2509	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2510	{
2511	/*
2512	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2513	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2514	*
2515	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2516	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2517	*/
2518	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2519	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2520	if (!fWasCached)
2521	{
2522	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2523	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2524	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2525	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2526	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2527	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2528	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2529	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2530	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2531	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2532	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2533	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2534	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2535	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2536	pVmcbNstGstCache->fCacheValid = true;
2537	Log4Func(("Cached VMCB fields\n"));
2538	}
2539
2540	return fWasCached;
2541	}
2542
2543
2544	/**
2545	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2546	*
2547	* This is done the first time we enter nested-guest execution using SVM R0
2548	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2549	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2550	*
2551	* @param pVCpu The cross context virtual CPU structure.
2552	*/
2553	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2554	{
2555	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2556	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2557
2558	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2559
2560	/*
2561	* First cache the nested-guest VMCB fields we may potentially modify.
2562	*/
2563	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2564	if (!fVmcbCached)
2565	{
2566	/*
2567	* The IOPM of the nested-guest can be ignored because the the guest always
2568	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2569	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2570	*/
2571	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2572
2573	/*
2574	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2575	* nested-paging suddenly while executing a VM (see assertion at the end of
2576	* Trap0eHandler() in PGMAllBth.h).
2577	*/
2578	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2579
2580	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2581	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2582
2583	/*
2584	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2585	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2586	*/
2587	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2588
2589	# ifdef DEBUG_ramshankar
2590	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2591	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2592	# endif
2593
2594	/*
2595	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2596	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2597	*/
2598	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2599	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2600	\| SVM_CTRL_INTERCEPT_VMLOAD;
2601
2602	/*
2603	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2604	* CLGI/STGI instructions executed by the nested-guest.
2605	*/
2606	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2607	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2608	\| SVM_CTRL_INTERCEPT_STGI;
2609
2610	/* Merge the guest and nested-guest intercepts. */
2611	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2612
2613	/* Update the VMCB clean bits. */
2614	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2615	}
2616	else
2617	{
2618	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2619	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2620	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2621	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2622	}
2623	}
2624
2625	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2626
2627	/**
2628	* Exports the state shared between the host and guest (or nested-guest) into
2629	* the VMCB.
2630	*
2631	* @param pVCpu The cross context virtual CPU structure.
2632	* @param pVmcb Pointer to the VM control block.
2633	*
2634	* @remarks No-long-jump zone!!!
2635	*/
2636	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2637	{
2638	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2639	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2640
2641	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2642	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2643
2644	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2645	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2646	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2647	}
2648
2649
2650	/**
2651	* Worker for SVMR0ImportStateOnDemand.
2652	*
2653	* @param pVCpu The cross context virtual CPU structure.
2654	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2655	*/
2656	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2657	{
2658	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2659
2660	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2661	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2662	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2663	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2664
2665	/*
2666	* We disable interrupts to make the updating of the state and in particular
2667	* the fExtrn modification atomic wrt to preemption hooks.
2668	*/
2669	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2670
2671	fWhat &= pCtx->fExtrn;
2672	if (fWhat)
2673	{
2674	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2675	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2676	{
2677	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2678	{
2679	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2680	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2681	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2682	}
2683	}
2684
2685	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2686	{
2687	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2688	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2689	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2690	}
2691	#endif
2692
2693	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2694	CPUMUpdateInterruptShadowEx(pCtx, pVmcbCtrl->IntShadow.n.u1IntShadow, pVmcbGuest->u64RIP);
2695
2696	if (fWhat & CPUMCTX_EXTRN_RIP)
2697	pCtx->rip = pVmcbGuest->u64RIP;
2698
2699	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2700	{
2701	pCtx->eflags.u = pVmcbGuest->u64RFlags;
2702	if (pVCpu->hmr0.s.fClearTrapFlag)
2703	{
2704	pVCpu->hmr0.s.fClearTrapFlag = false;
2705	pCtx->eflags.Bits.u1TF = 0;
2706	}
2707	}
2708
2709	if (fWhat & CPUMCTX_EXTRN_RSP)
2710	pCtx->rsp = pVmcbGuest->u64RSP;
2711
2712	if (fWhat & CPUMCTX_EXTRN_RAX)
2713	pCtx->rax = pVmcbGuest->u64RAX;
2714
2715	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2716	{
2717	if (fWhat & CPUMCTX_EXTRN_CS)
2718	{
2719	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2720	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2721	/** @todo SELM might need to be fixed as it too should not care about the
2722	* granularity bit. See @bugref{6785}. */
2723	if ( !pCtx->cs.Attr.n.u1Granularity
2724	&& pCtx->cs.Attr.n.u1Present
2725	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2726	{
2727	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2728	pCtx->cs.Attr.n.u1Granularity = 1;
2729	}
2730	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2731	}
2732	if (fWhat & CPUMCTX_EXTRN_SS)
2733	{
2734	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2735	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2736	/*
2737	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2738	* VMCB and uses that and thus it's possible that when the CPL changes during
2739	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2740	* AMD Fusion CPUs with 64-bit guests.
2741	*
2742	* See AMD spec. 15.5.1 "Basic operation".
2743	*/
2744	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2745	uint8_t const uCpl = pVmcbGuest->u8CPL;
2746	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2747	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2748	}
2749	if (fWhat & CPUMCTX_EXTRN_DS)
2750	{
2751	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2752	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2753	}
2754	if (fWhat & CPUMCTX_EXTRN_ES)
2755	{
2756	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2757	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2758	}
2759	if (fWhat & CPUMCTX_EXTRN_FS)
2760	{
2761	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2762	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2763	}
2764	if (fWhat & CPUMCTX_EXTRN_GS)
2765	{
2766	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2767	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2768	}
2769	}
2770
2771	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2772	{
2773	if (fWhat & CPUMCTX_EXTRN_TR)
2774	{
2775	/*
2776	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2777	* are used between Intel and AMD, see @bugref{6208#c39}.
2778	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2779	*/
2780	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2781	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2782	{
2783	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2784	\|\| CPUMIsGuestInLongModeEx(pCtx))
2785	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2786	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2787	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2788	}
2789	}
2790
2791	if (fWhat & CPUMCTX_EXTRN_LDTR)
2792	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2793
2794	if (fWhat & CPUMCTX_EXTRN_GDTR)
2795	{
2796	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2797	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2798	}
2799
2800	if (fWhat & CPUMCTX_EXTRN_IDTR)
2801	{
2802	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2803	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2804	}
2805	}
2806
2807	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2808	{
2809	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2810	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2811	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2812	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2813	}
2814
2815	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2816	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2817	{
2818	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2819	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2820	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2821	}
2822
2823	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2824	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2825
2826	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2827	{
2828	if (fWhat & CPUMCTX_EXTRN_DR6)
2829	{
2830	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2831	pCtx->dr[6] = pVmcbGuest->u64DR6;
2832	else
2833	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2834	}
2835
2836	if (fWhat & CPUMCTX_EXTRN_DR7)
2837	{
2838	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2839	pCtx->dr[7] = pVmcbGuest->u64DR7;
2840	else
2841	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2842	}
2843	}
2844
2845	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2846	{
2847	if (fWhat & CPUMCTX_EXTRN_CR0)
2848	{
2849	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2850	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2851	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2852	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2853	CPUMSetGuestCR0(pVCpu, uCr0);
2854	VMMRZCallRing3Enable(pVCpu);
2855	}
2856
2857	if (fWhat & CPUMCTX_EXTRN_CR2)
2858	pCtx->cr2 = pVmcbGuest->u64CR2;
2859
2860	if (fWhat & CPUMCTX_EXTRN_CR3)
2861	{
2862	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2863	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2864	{
2865	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2866	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2867	}
2868	}
2869
2870	/* Changes to CR4 are always intercepted. */
2871	}
2872
2873	/* Update fExtrn. */
2874	pCtx->fExtrn &= ~fWhat;
2875
2876	/* If everything has been imported, clear the HM keeper bit. */
2877	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2878	{
2879	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2880	Assert(!pCtx->fExtrn);
2881	}
2882	}
2883	else
2884	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2885
2886	ASMSetFlags(fEFlags);
2887
2888	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2889
2890	/*
2891	* Honor any pending CR3 updates.
2892	*
2893	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2894	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2895	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2896	* handling -> hmR0SvmImportGuestState() and here we are.
2897	*
2898	* The reason for such complicated handling is because VM-exits that call into PGM expect
2899	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2900	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2901	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2902	*
2903	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2904	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2905	*/
2906	if ( VMMRZCallRing3IsEnabled(pVCpu)
2907	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2908	{
2909	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2910	PGMUpdateCR3(pVCpu, pCtx->cr3);
2911	}
2912	}
2913
2914
2915	/**
2916	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2917	* context.
2918	*
2919	* Currently there is no residual state left in the CPU that is not updated in the
2920	* VMCB.
2921	*
2922	* @returns VBox status code.
2923	* @param pVCpu The cross context virtual CPU structure.
2924	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2925	*/
2926	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2927	{
2928	hmR0SvmImportGuestState(pVCpu, fWhat);
2929	return VINF_SUCCESS;
2930	}
2931
2932
2933	/**
2934	* Gets SVM \#VMEXIT auxiliary information.
2935	*
2936	* @returns VBox status code.
2937	* @param pVCpu The cross context virtual CPU structure.
2938	* @param pSvmExitAux Where to store the auxiliary info.
2939	*/
2940	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2941	{
2942	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2943	if (RT_LIKELY(pSvmTransient))
2944	{
2945	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2946	if (RT_LIKELY(pVmcb))
2947	{
2948	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2949	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2950	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2951	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2952	return VINF_SUCCESS;
2953	}
2954	return VERR_SVM_IPE_5;
2955	}
2956	return VERR_NOT_AVAILABLE;
2957	}
2958
2959
2960	/**
2961	* Does the necessary state syncing before returning to ring-3 for any reason
2962	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2963	*
2964	* @param pVCpu The cross context virtual CPU structure.
2965	* @param fImportState Whether to import the guest state from the VMCB back
2966	* to the guest-CPU context.
2967	*
2968	* @remarks No-long-jmp zone!!!
2969	*/
2970	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2971	{
2972	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2973	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2974
2975	/*
2976	* !!! IMPORTANT !!!
2977	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2978	*/
2979
2980	/* Save the guest state if necessary. */
2981	if (fImportState)
2982	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2983
2984	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2985	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2986	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2987
2988	/*
2989	* Restore host debug registers if necessary and resync on next R0 reentry.
2990	*/
2991	#ifdef VBOX_STRICT
2992	if (CPUMIsHyperDebugStateActive(pVCpu))
2993	{
2994	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2995	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2996	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2997	}
2998	#endif
2999	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3000	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3001	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3002
3003	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3004	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3005	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3006	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3007	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3008	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3009	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3010
3011	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3012	}
3013
3014
3015	/**
3016	* Leaves the AMD-V session.
3017	*
3018	* Only used while returning to ring-3 either due to longjump or exits to
3019	* ring-3.
3020	*
3021	* @returns VBox status code.
3022	* @param pVCpu The cross context virtual CPU structure.
3023	*/
3024	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3025	{
3026	HM_DISABLE_PREEMPT(pVCpu);
3027	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3028	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3029
3030	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3031	and done this from the SVMR0ThreadCtxCallback(). */
3032	if (!pVCpu->hmr0.s.fLeaveDone)
3033	{
3034	hmR0SvmLeave(pVCpu, true /* fImportState */);
3035	pVCpu->hmr0.s.fLeaveDone = true;
3036	}
3037
3038	/*
3039	* !!! IMPORTANT !!!
3040	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3041	*/
3042
3043	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3044	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3045	VMMR0ThreadCtxHookDisable(pVCpu);
3046
3047	/* Leave HM context. This takes care of local init (term). */
3048	int rc = HMR0LeaveCpu(pVCpu);
3049
3050	HM_RESTORE_PREEMPT();
3051	return rc;
3052	}
3053
3054
3055	/**
3056	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3057	* any remaining host state) before we go back to ring-3 due to an assertion.
3058	*
3059	* @param pVCpu The cross context virtual CPU structure.
3060	*/
3061	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3062	{
3063	/*
3064	* !!! IMPORTANT !!!
3065	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3066	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3067	*/
3068	VMMR0AssertionRemoveNotification(pVCpu);
3069	VMMRZCallRing3Disable(pVCpu);
3070	HM_DISABLE_PREEMPT(pVCpu);
3071
3072	/* Import the entire guest state. */
3073	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3074
3075	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3076	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3077
3078	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3079	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3080
3081	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3082	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3083	VMMR0ThreadCtxHookDisable(pVCpu);
3084
3085	/* Leave HM context. This takes care of local init (term). */
3086	HMR0LeaveCpu(pVCpu);
3087
3088	HM_RESTORE_PREEMPT();
3089	return VINF_SUCCESS;
3090	}
3091
3092
3093	/**
3094	* Take necessary actions before going back to ring-3.
3095	*
3096	* An action requires us to go back to ring-3. This function does the necessary
3097	* steps before we can safely return to ring-3. This is not the same as longjmps
3098	* to ring-3, this is voluntary.
3099	*
3100	* @returns Strict VBox status code.
3101	* @param pVCpu The cross context virtual CPU structure.
3102	* @param rcExit The reason for exiting to ring-3. Can be
3103	* VINF_VMM_UNKNOWN_RING3_CALL.
3104	*/
3105	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3106	{
3107	Assert(pVCpu);
3108	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3109
3110	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3111	VMMRZCallRing3Disable(pVCpu);
3112	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3113	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3114
3115	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3116	if (pVCpu->hm.s.Event.fPending)
3117	{
3118	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3119	Assert(!pVCpu->hm.s.Event.fPending);
3120	}
3121
3122	/* Sync. the necessary state for going back to ring-3. */
3123	hmR0SvmLeaveSession(pVCpu);
3124	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3125
3126	/* Thread-context hooks are unregistered at this point!!! */
3127	/* Ring-3 callback notifications are unregistered at this point!!! */
3128
3129	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3130	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3131	\| CPUM_CHANGED_LDTR
3132	\| CPUM_CHANGED_GDTR
3133	\| CPUM_CHANGED_IDTR
3134	\| CPUM_CHANGED_TR
3135	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3136	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3137	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3138	{
3139	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3140	}
3141
3142	/* Update the exit-to-ring 3 reason. */
3143	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3144
3145	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3146	if ( rcExit != VINF_EM_RAW_INTERRUPT
3147	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3148	{
3149	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3150	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3151	}
3152
3153	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3154	VMMRZCallRing3Enable(pVCpu);
3155
3156	/*
3157	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3158	* and if we're injecting an event we should have a TRPM trap pending.
3159	*/
3160	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3161	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3162	VERR_SVM_IPE_5);
3163	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3164	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3165	VERR_SVM_IPE_4);
3166
3167	return rcExit;
3168	}
3169
3170
3171	/**
3172	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3173	* intercepts.
3174	*
3175	* @param pVCpu The cross context virtual CPU structure.
3176	* @param pVmcb Pointer to the VM control block.
3177	*
3178	* @remarks No-long-jump zone!!!
3179	*/
3180	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3181	{
3182	/*
3183	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3184	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3185	* RDTSC/RDTSCP as well.
3186	*/
3187	bool fParavirtTsc;
3188	uint64_t uTscOffset;
3189	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3190
3191	bool fIntercept;
3192	if (fCanUseRealTsc)
3193	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3194	else
3195	{
3196	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3197	fIntercept = true;
3198	}
3199
3200	if (!fIntercept)
3201	{
3202	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3203	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3204	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3205	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3206	#endif
3207
3208	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3209	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3210	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3211	}
3212
3213	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3214	information before every VM-entry, hence we have nothing to do here at the moment. */
3215	if (fParavirtTsc)
3216	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3217	}
3218
3219
3220	/**
3221	* Sets an event as a pending event to be injected into the guest.
3222	*
3223	* @param pVCpu The cross context virtual CPU structure.
3224	* @param pEvent Pointer to the SVM event.
3225	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3226	* page-fault.
3227	*
3228	* @remarks Statistics counter assumes this is a guest event being reflected to
3229	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3230	*/
3231	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3232	{
3233	Assert(!pVCpu->hm.s.Event.fPending);
3234	Assert(pEvent->n.u1Valid);
3235
3236	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3237	pVCpu->hm.s.Event.fPending = true;
3238	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3239
3240	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3241	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3242	}
3243
3244
3245	/**
3246	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3247	*
3248	* @param pVCpu The cross context virtual CPU structure.
3249	*/
3250	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3251	{
3252	SVMEVENT Event;
3253	Event.u = 0;
3254	Event.n.u1Valid = 1;
3255	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3256	Event.n.u8Vector = X86_XCPT_DE;
3257	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3258	}
3259
3260
3261	/**
3262	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3263	*
3264	* @param pVCpu The cross context virtual CPU structure.
3265	*/
3266	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3267	{
3268	SVMEVENT Event;
3269	Event.u = 0;
3270	Event.n.u1Valid = 1;
3271	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3272	Event.n.u8Vector = X86_XCPT_UD;
3273	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3274	}
3275
3276
3277	/**
3278	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3279	*
3280	* @param pVCpu The cross context virtual CPU structure.
3281	*/
3282	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3283	{
3284	SVMEVENT Event;
3285	Event.u = 0;
3286	Event.n.u1Valid = 1;
3287	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3288	Event.n.u8Vector = X86_XCPT_DB;
3289	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3290	}
3291
3292
3293	/**
3294	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3295	*
3296	* @param pVCpu The cross context virtual CPU structure.
3297	* @param u32ErrCode The error-code for the page-fault.
3298	* @param uFaultAddress The page fault address (CR2).
3299	*
3300	* @remarks This updates the guest CR2 with @a uFaultAddress!
3301	*/
3302	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3303	{
3304	SVMEVENT Event;
3305	Event.u = 0;
3306	Event.n.u1Valid = 1;
3307	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3308	Event.n.u8Vector = X86_XCPT_PF;
3309	Event.n.u1ErrorCodeValid = 1;
3310	Event.n.u32ErrorCode = u32ErrCode;
3311
3312	/* Update CR2 of the guest. */
3313	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3314	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3315	{
3316	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3317	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3318	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3319	}
3320
3321	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3322	}
3323
3324
3325	/**
3326	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3327	*
3328	* @param pVCpu The cross context virtual CPU structure.
3329	*/
3330	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3331	{
3332	SVMEVENT Event;
3333	Event.u = 0;
3334	Event.n.u1Valid = 1;
3335	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3336	Event.n.u8Vector = X86_XCPT_MF;
3337	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3338	}
3339
3340
3341	/**
3342	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3343	*
3344	* @param pVCpu The cross context virtual CPU structure.
3345	*/
3346	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3347	{
3348	SVMEVENT Event;
3349	Event.u = 0;
3350	Event.n.u1Valid = 1;
3351	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3352	Event.n.u8Vector = X86_XCPT_DF;
3353	Event.n.u1ErrorCodeValid = 1;
3354	Event.n.u32ErrorCode = 0;
3355	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3356	}
3357
3358
3359	/**
3360	* Injects an event into the guest upon VMRUN by updating the relevant field
3361	* in the VMCB.
3362	*
3363	* @param pVCpu The cross context virtual CPU structure.
3364	* @param pVmcb Pointer to the guest VM control block.
3365	* @param pEvent Pointer to the event.
3366	*
3367	* @remarks No-long-jump zone!!!
3368	* @remarks Requires CR0!
3369	*/
3370	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3371	{
3372	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3373	pVmcb->ctrl.EventInject.u = pEvent->u;
3374	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3375	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3376	{
3377	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3378	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3379	}
3380	else
3381	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3382	RT_NOREF(pVCpu);
3383
3384	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3385	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3386	}
3387
3388
3389
3390	/**
3391	* Converts any TRPM trap into a pending HM event. This is typically used when
3392	* entering from ring-3 (not longjmp returns).
3393	*
3394	* @param pVCpu The cross context virtual CPU structure.
3395	*/
3396	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3397	{
3398	Assert(TRPMHasTrap(pVCpu));
3399	Assert(!pVCpu->hm.s.Event.fPending);
3400
3401	uint8_t uVector;
3402	TRPMEVENT enmTrpmEvent;
3403	uint32_t uErrCode;
3404	RTGCUINTPTR GCPtrFaultAddress;
3405	uint8_t cbInstr;
3406
3407	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3408	AssertRC(rc);
3409
3410	SVMEVENT Event;
3411	Event.u = 0;
3412	Event.n.u1Valid = 1;
3413	Event.n.u8Vector = uVector;
3414
3415	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3416	if (enmTrpmEvent == TRPM_TRAP)
3417	{
3418	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3419	switch (uVector)
3420	{
3421	case X86_XCPT_NMI:
3422	{
3423	Event.n.u3Type = SVM_EVENT_NMI;
3424	break;
3425	}
3426
3427	case X86_XCPT_BP:
3428	case X86_XCPT_OF:
3429	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3430	RT_FALL_THRU();
3431
3432	case X86_XCPT_PF:
3433	case X86_XCPT_DF:
3434	case X86_XCPT_TS:
3435	case X86_XCPT_NP:
3436	case X86_XCPT_SS:
3437	case X86_XCPT_GP:
3438	case X86_XCPT_AC:
3439	{
3440	Event.n.u1ErrorCodeValid = 1;
3441	Event.n.u32ErrorCode = uErrCode;
3442	break;
3443	}
3444	}
3445	}
3446	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3447	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3448	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3449	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3450	else
3451	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3452
3453	rc = TRPMResetTrap(pVCpu);
3454	AssertRC(rc);
3455
3456	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3457	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3458
3459	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3460	}
3461
3462
3463	/**
3464	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3465	* AMD-V to execute any instruction.
3466	*
3467	* @param pVCpu The cross context virtual CPU structure.
3468	*/
3469	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3470	{
3471	Assert(pVCpu->hm.s.Event.fPending);
3472	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3473
3474	SVMEVENT Event;
3475	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3476
3477	uint8_t uVector = Event.n.u8Vector;
3478	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3479
3480	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3481
3482	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3483	AssertRC(rc);
3484
3485	if (Event.n.u1ErrorCodeValid)
3486	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3487
3488	if ( enmTrapType == TRPM_TRAP
3489	&& uVector == X86_XCPT_PF)
3490	{
3491	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3492	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3493	}
3494	else if (enmTrapType == TRPM_SOFTWARE_INT)
3495	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3496	pVCpu->hm.s.Event.fPending = false;
3497	}
3498
3499
3500	/**
3501	* Sets the virtual interrupt intercept control in the VMCB.
3502	*
3503	* @param pVCpu The cross context virtual CPU structure.
3504	* @param pVmcb Pointer to the VM control block.
3505	*/
3506	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3507	{
3508	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3509
3510	/*
3511	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3512	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3513	* (updating ISR at the right time) and inject the interrupt.
3514	*
3515	* With AVIC is supported, we could make use of the asynchronously delivery without
3516	* #VMEXIT and we would be passing the AVIC page to SVM.
3517	*
3518	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3519	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3520	*/
3521	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3522	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3523	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3524	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3525	Log4(("Set VINTR intercept\n"));
3526	}
3527
3528
3529	/**
3530	* Clears the virtual interrupt intercept control in the VMCB as
3531	* we are figured the guest is unable process any interrupts
3532	* at this point of time.
3533	*
3534	* @param pVCpu The cross context virtual CPU structure.
3535	* @param pVmcb Pointer to the VM control block.
3536	*/
3537	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3538	{
3539	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3540
3541	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3542	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3543	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3544	{
3545	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3546	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3547	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3548	Log4(("Cleared VINTR intercept\n"));
3549	}
3550	}
3551
3552
3553	/**
3554	* Evaluates the event to be delivered to the guest and sets it as the pending
3555	* event.
3556	*
3557	* @returns Strict VBox status code.
3558	* @param pVCpu The cross context virtual CPU structure.
3559	* @param pSvmTransient Pointer to the SVM transient structure.
3560	*/
3561	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3562	{
3563	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3564	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3565	\| CPUMCTX_EXTRN_RFLAGS
3566	\| CPUMCTX_EXTRN_INHIBIT_INT
3567	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3568
3569	Assert(!pVCpu->hm.s.Event.fPending);
3570	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3571	Assert(pVmcb);
3572
3573	bool const fGif = CPUMGetGuestGif(pCtx);
3574	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(pCtx);
3575	bool const fBlockNmi = CPUMAreInterruptsInhibitedByNmi(pCtx);
3576
3577	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3578	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3579	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3580
3581	/** @todo SMI. SMIs take priority over NMIs. */
3582
3583	/*
3584	* Check if the guest or nested-guest can receive NMIs.
3585	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3586	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3587	*/
3588	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3589	&& !fBlockNmi)
3590	{
3591	if ( fGif
3592	&& !fIntShadow)
3593	{
3594	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3595	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3596	{
3597	Log4(("Intercepting NMI -> #VMEXIT\n"));
3598	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3599	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3600	}
3601	#endif
3602	Log4(("Setting NMI pending for injection\n"));
3603	SVMEVENT Event;
3604	Event.u = 0;
3605	Event.n.u1Valid = 1;
3606	Event.n.u8Vector = X86_XCPT_NMI;
3607	Event.n.u3Type = SVM_EVENT_NMI;
3608	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3609	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3610	}
3611	else if (!fGif)
3612	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3613	else if (!pSvmTransient->fIsNestedGuest)
3614	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3615	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3616	}
3617	/*
3618	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3619	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3620	* it from the APIC device.
3621	*
3622	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3623	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3624	* do that work when we execute nested-guest code esp. since all the required information
3625	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3626	* the virtual interrupt controller.
3627	*
3628	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3629	*/
3630	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3631	&& !pVCpu->hm.s.fSingleInstruction)
3632	{
3633	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u & X86_EFL_IF)
3634	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3635	if ( fGif
3636	&& !fBlockInt
3637	&& !fIntShadow)
3638	{
3639	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3640	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3641	{
3642	Log4(("Intercepting INTR -> #VMEXIT\n"));
3643	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3644	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3645	}
3646	#endif
3647	uint8_t u8Interrupt;
3648	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3649	if (RT_SUCCESS(rc))
3650	{
3651	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3652	SVMEVENT Event;
3653	Event.u = 0;
3654	Event.n.u1Valid = 1;
3655	Event.n.u8Vector = u8Interrupt;
3656	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3657	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3658	}
3659	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3660	{
3661	/*
3662	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3663	* updated eventually when the TPR is written by the guest.
3664	*/
3665	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3666	}
3667	else
3668	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3669	}
3670	else if (!fGif)
3671	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3672	else if (!pSvmTransient->fIsNestedGuest)
3673	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3674	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3675	}
3676
3677	return VINF_SUCCESS;
3678	}
3679
3680
3681	/**
3682	* Injects any pending events into the guest (or nested-guest).
3683	*
3684	* @param pVCpu The cross context virtual CPU structure.
3685	* @param pVmcb Pointer to the VM control block.
3686	*
3687	* @remarks Must only be called when we are guaranteed to enter
3688	* hardware-assisted SVM execution and not return to ring-3
3689	* prematurely.
3690	*/
3691	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3692	{
3693	Assert(!TRPMHasTrap(pVCpu));
3694	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3695
3696	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx);
3697	#ifdef VBOX_STRICT
3698	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3699	bool const fGif = CPUMGetGuestGif(pCtx);
3700	bool fAllowInt = fGif;
3701	if (fGif)
3702	{
3703	/*
3704	* For nested-guests we have no way to determine if we're injecting a physical or
3705	* virtual interrupt at this point. Hence the partial verification below.
3706	*/
3707	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3708	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3709	else
3710	fAllowInt = RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
3711	}
3712	#endif
3713
3714	if (pVCpu->hm.s.Event.fPending)
3715	{
3716	SVMEVENT Event;
3717	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3718	Assert(Event.n.u1Valid);
3719
3720	/*
3721	* Validate event injection pre-conditions.
3722	*/
3723	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3724	{
3725	Assert(fAllowInt);
3726	Assert(!fIntShadow);
3727	}
3728	else if (Event.n.u3Type == SVM_EVENT_NMI)
3729	{
3730	Assert(fGif);
3731	Assert(!fIntShadow);
3732	}
3733
3734	/*
3735	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3736	* do this only when we are surely going to inject the NMI as otherwise if we return
3737	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3738	* SVM R0.
3739	*
3740	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3741	* which will set the VMCS field after actually delivering the NMI which we read on
3742	* VM-exit to determine the state.
3743	*/
3744	if ( Event.n.u3Type == SVM_EVENT_NMI
3745	&& Event.n.u8Vector == X86_XCPT_NMI)
3746	CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3747
3748	/*
3749	* Inject it (update VMCB for injection by the hardware).
3750	*/
3751	Log4(("Injecting pending HM event\n"));
3752	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3753	pVCpu->hm.s.Event.fPending = false;
3754
3755	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3756	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3757	else
3758	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3759	}
3760	else
3761	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3762
3763	/*
3764	* We could have injected an NMI through IEM and continue guest execution using
3765	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3766	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3767	*/
3768	if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
3769	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3770
3771	/*
3772	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3773	*
3774	* For nested-guests: We need to update it too for the scenario where IEM executes
3775	* the nested-guest but execution later continues here with an interrupt shadow active.
3776	*/
3777	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3778	}
3779
3780
3781	/**
3782	* Reports world-switch error and dumps some useful debug info.
3783	*
3784	* @param pVCpu The cross context virtual CPU structure.
3785	* @param rcVMRun The return code from VMRUN (or
3786	* VERR_SVM_INVALID_GUEST_STATE for invalid
3787	* guest-state).
3788	*/
3789	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3790	{
3791	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3792	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3793	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3794
3795	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3796	{
3797	#ifdef VBOX_STRICT
3798	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3799	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3800	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3801	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3802	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3803	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3804	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3805	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3806	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3807	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3808	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3809	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3810
3811	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3812	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3813	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3814
3815	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3816	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3817	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3818	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3819	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3820	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3821	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3822	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3823	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3824	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3825	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3826	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3827
3828	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3829	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3830	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3831	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3832	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3833	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3834	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3835	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3836	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3837	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3838	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3839	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3840	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3841	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3842	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3843	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3844	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3845	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3846	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3847	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3848
3849	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3850
3851	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3852	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3853
3854	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3855	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3856	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3857	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3858	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3859	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3860	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3861	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3862	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3863	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3864	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3865	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3866	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3867	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3868	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3869	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3870	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3871	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3872	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3873	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3874
3875	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3876	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3877
3878	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3879	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3880	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3881	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3882
3883	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3884	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3885
3886	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3887	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3888	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3889	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3890
3891	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3892	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3893	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3894	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3895	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3896	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3897	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3898
3899	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3900	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3901	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3902	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3903
3904	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3905	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3906	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3907
3908	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3909	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3910	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3911	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3912	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3913	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3914	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3915	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3916	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3917	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3918	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3919	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3920
3921	NOREF(pVmcb);
3922	#endif /* VBOX_STRICT */
3923	}
3924	else
3925	Log4Func(("rcVMRun=%d\n", rcVMRun));
3926	}
3927
3928
3929	/**
3930	* Check per-VM and per-VCPU force flag actions that require us to go back to
3931	* ring-3 for one reason or another.
3932	*
3933	* @returns Strict VBox status code (information status code included).
3934	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3935	* ring-3.
3936	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3937	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3938	* interrupts)
3939	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3940	* all EMTs to be in ring-3.
3941	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3942	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3943	* to the EM loop.
3944	*
3945	* @param pVCpu The cross context virtual CPU structure.
3946	*/
3947	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3948	{
3949	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3950
3951	/* Could happen as a result of longjump. */
3952	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3953	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3954
3955	/* Update pending interrupts into the APIC's IRR. */
3956	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3957	APICUpdatePendingInterrupts(pVCpu);
3958
3959	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3960	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3961	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3962	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3963	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3964	{
3965	/* Pending PGM C3 sync. */
3966	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3967	{
3968	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3969	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3970	if (rc != VINF_SUCCESS)
3971	{
3972	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3973	return rc;
3974	}
3975	}
3976
3977	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3978	/* -XXX- what was that about single stepping? */
3979	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3980	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3981	{
3982	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3983	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3984	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3985	return rc;
3986	}
3987
3988	/* Pending VM request packets, such as hardware interrupts. */
3989	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3990	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3991	{
3992	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
3993	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3994	return VINF_EM_PENDING_REQUEST;
3995	}
3996
3997	/* Pending PGM pool flushes. */
3998	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3999	{
4000	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4001	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4002	return VINF_PGM_POOL_FLUSH_PENDING;
4003	}
4004
4005	/* Pending DMA requests. */
4006	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4007	{
4008	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4009	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4010	return VINF_EM_RAW_TO_R3;
4011	}
4012	}
4013
4014	return VINF_SUCCESS;
4015	}
4016
4017
4018	/**
4019	* Does the preparations before executing guest code in AMD-V.
4020	*
4021	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4022	* recompiler. We must be cautious what we do here regarding committing
4023	* guest-state information into the VMCB assuming we assuredly execute the guest
4024	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4025	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4026	* that the recompiler can (and should) use them when it resumes guest
4027	* execution. Otherwise such operations must be done when we can no longer
4028	* exit to ring-3.
4029	*
4030	* @returns Strict VBox status code (informational status codes included).
4031	* @retval VINF_SUCCESS if we can proceed with running the guest.
4032	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4033	*
4034	* @param pVCpu The cross context virtual CPU structure.
4035	* @param pSvmTransient Pointer to the SVM transient structure.
4036	*/
4037	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4038	{
4039	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4040
4041	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4042	if (pSvmTransient->fIsNestedGuest)
4043	{
4044	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4045	return VINF_EM_RESCHEDULE_REM;
4046	}
4047	#endif
4048
4049	/* Check force flag actions that might require us to go back to ring-3. */
4050	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4051	if (rc != VINF_SUCCESS)
4052	return rc;
4053
4054	if (TRPMHasTrap(pVCpu))
4055	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4056	else if (!pVCpu->hm.s.Event.fPending)
4057	{
4058	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4059	if ( rc != VINF_SUCCESS
4060	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4061	{
4062	/* If a nested-guest VM-exit occurred, bail. */
4063	if (pSvmTransient->fIsNestedGuest)
4064	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4065	return rc;
4066	}
4067	}
4068
4069	/*
4070	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4071	* Just do it in software, see @bugref{8411}.
4072	* NB: If we could continue a task switch exit we wouldn't need to do this.
4073	*/
4074	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4075	if (RT_UNLIKELY( !g_fHmSvmFeatures
4076	&& pVCpu->hm.s.Event.fPending
4077	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4078	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4079
4080	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4081	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4082	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4083	#endif
4084
4085	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4086	/*
4087	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4088	*/
4089	if (pSvmTransient->fIsNestedGuest)
4090	hmR0SvmSetupVmcbNested(pVCpu);
4091	#endif
4092
4093	/*
4094	* Export the guest state bits that are not shared with the host in any way as we can
4095	* longjmp or get preempted in the midst of exporting some of the state.
4096	*/
4097	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4098	AssertRCReturn(rc, rc);
4099	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4100
4101	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4102	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4103
4104	/*
4105	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4106	* world-switch so we can update it on the way back if the guest changed the TPR.
4107	*/
4108	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4109	{
4110	Assert(!pSvmTransient->fIsNestedGuest);
4111	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4112	if (pVM->hm.s.fTprPatchingActive)
4113	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4114	else
4115	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4116	}
4117
4118	/*
4119	* No longjmps to ring-3 from this point on!!!
4120	*
4121	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4122	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4123	*/
4124	VMMRZCallRing3Disable(pVCpu);
4125
4126	/*
4127	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4128	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4129	* preemption disabled for a while. Since this is purly to aid the
4130	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4131	* disable interrupt on NT.
4132	*
4133	* We need to check for force-flags that could've possible been altered since we last
4134	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4135	* see @bugref{6398}).
4136	*
4137	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4138	* to ring-3 before executing guest code.
4139	*/
4140	pSvmTransient->fEFlags = ASMIntDisableFlags();
4141	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4142	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4143	{
4144	ASMSetFlags(pSvmTransient->fEFlags);
4145	VMMRZCallRing3Enable(pVCpu);
4146	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4147	return VINF_EM_RAW_TO_R3;
4148	}
4149	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4150	{
4151	ASMSetFlags(pSvmTransient->fEFlags);
4152	VMMRZCallRing3Enable(pVCpu);
4153	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4154	return VINF_EM_RAW_INTERRUPT;
4155	}
4156
4157	return VINF_SUCCESS;
4158	}
4159
4160
4161	/**
4162	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4163	* doing so.
4164	*
4165	* This means there is no backing out to ring-3 or anywhere else at this point.
4166	*
4167	* @param pVCpu The cross context virtual CPU structure.
4168	* @param pSvmTransient Pointer to the SVM transient structure.
4169	*
4170	* @remarks Called with preemption disabled.
4171	* @remarks No-long-jump zone!!!
4172	*/
4173	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4174	{
4175	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4176	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4177
4178	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4179	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4180
4181	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4182	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4183
4184	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4185
4186	if (!CPUMIsGuestFPUStateActive(pVCpu))
4187	{
4188	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4189	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4190	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4191	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4192	}
4193
4194	/* Load the state shared between host and guest (FPU, debug). */
4195	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4196	hmR0SvmExportSharedState(pVCpu, pVmcb);
4197
4198	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4199	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4200
4201	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4202	RTCPUID const idHostCpu = pHostCpu->idCpu;
4203	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4204
4205	/* Setup TSC offsetting. */
4206	if ( pSvmTransient->fUpdateTscOffsetting
4207	\|\| fMigratedHostCpu)
4208	{
4209	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4210	pSvmTransient->fUpdateTscOffsetting = false;
4211	}
4212
4213	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4214	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4215	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4216	else
4217	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4218
4219	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4220	if (fMigratedHostCpu)
4221	pVmcb->ctrl.u32VmcbCleanBits = 0;
4222
4223	/* Store status of the shared guest-host state at the time of VMRUN. */
4224	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4225	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4226
4227	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4228	uint8_t *pbMsrBitmap;
4229	if (!pSvmTransient->fIsNestedGuest)
4230	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4231	else
4232	{
4233	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4234	* MSRPM and only perform this if it changed also use EVEX.POR when it
4235	* does. */
4236	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4237
4238	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4239	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4240	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4241	}
4242	#else
4243	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4244	#endif
4245
4246	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4247	/* Flush the appropriate tagged-TLB entries. */
4248	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4249	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4250
4251	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4252
4253	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4254	to start executing. */
4255
4256	/*
4257	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4258	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4259	*
4260	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4261	*/
4262	if ( g_CpumHostFeatures.s.fRdTscP
4263	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4264	{
4265	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4266	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4267	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4268	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4269	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4270	pSvmTransient->fRestoreTscAuxMsr = true;
4271	}
4272	else
4273	{
4274	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4275	pSvmTransient->fRestoreTscAuxMsr = false;
4276	}
4277	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4278
4279	/*
4280	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4281	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4282	* from the VMCB.
4283	*/
4284	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4285	if (!fSupportsVmcbCleanBits)
4286	pVmcb->ctrl.u32VmcbCleanBits = 0;
4287	}
4288
4289
4290	/**
4291	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4292	*
4293	* @returns VBox strict status code.
4294	* @param pVCpu The cross context virtual CPU structure.
4295	* @param HCPhysVmcb The host physical address of the VMCB.
4296	*
4297	* @remarks No-long-jump zone!!!
4298	*/
4299	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4300	{
4301	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4302	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4303	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4304	}
4305
4306
4307	/**
4308	* Performs some essential restoration of state after running guest (or
4309	* nested-guest) code in AMD-V.
4310	*
4311	* @param pVCpu The cross context virtual CPU structure.
4312	* @param pSvmTransient Pointer to the SVM transient structure.
4313	* @param rcVMRun Return code of VMRUN.
4314	*
4315	* @remarks Called with interrupts disabled.
4316	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4317	* unconditionally when it is safe to do so.
4318	*/
4319	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4320	{
4321	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4322
4323	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4324	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4325
4326	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4327	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4328
4329	/* TSC read must be done early for maximum accuracy. */
4330	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4331	{
4332	if (!pSvmTransient->fIsNestedGuest)
4333	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4334	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4335	else
4336	{
4337	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4338	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4339	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4340	}
4341	#endif
4342	}
4343
4344	if (pSvmTransient->fRestoreTscAuxMsr)
4345	{
4346	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4347	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4348	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4349	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4350	}
4351
4352	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4353	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4354	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4355	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4356
4357	Assert(!(ASMGetFlags() & X86_EFL_IF));
4358	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4359	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4360
4361	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4362	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4363	{
4364	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4365	return;
4366	}
4367
4368	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4369	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4370	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4371	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4372
4373	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4374	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4375	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4376	#else
4377	/*
4378	* Always import the following:
4379	*
4380	* - RIP for exit optimizations and evaluating event injection on re-entry.
4381	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4382	* state on preemption.
4383	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4384	* - CS for exit optimizations.
4385	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4386	* assembly switcher code.
4387	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4388	*/
4389	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4390	\| CPUMCTX_EXTRN_RFLAGS
4391	\| CPUMCTX_EXTRN_RAX
4392	\| CPUMCTX_EXTRN_RSP
4393	\| CPUMCTX_EXTRN_CS
4394	\| CPUMCTX_EXTRN_HWVIRT
4395	\| CPUMCTX_EXTRN_INHIBIT_INT
4396	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4397	\| HMSVM_CPUMCTX_SHARED_STATE);
4398	#endif
4399
4400	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4401	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4402	{
4403	Assert(!pSvmTransient->fIsNestedGuest);
4404	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4405	if ( pVM->hm.s.fTprPatchingActive
4406	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4407	{
4408	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4409	AssertRC(rc);
4410	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4411	}
4412	/* Sync TPR when we aren't intercepting CR8 writes. */
4413	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4414	{
4415	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4416	AssertRC(rc);
4417	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4418	}
4419	}
4420
4421	#ifdef DEBUG_ramshankar
4422	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4423	{
4424	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4425	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4426	0 /* uVerbose */);
4427	}
4428	#endif
4429
4430	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4431	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4432	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4433	}
4434
4435
4436	/**
4437	* Runs the guest code using AMD-V.
4438	*
4439	* @returns Strict VBox status code.
4440	* @param pVCpu The cross context virtual CPU structure.
4441	* @param pcLoops Pointer to the number of executed loops.
4442	*/
4443	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4444	{
4445	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4446	Assert(pcLoops);
4447	Assert(*pcLoops <= cMaxResumeLoops);
4448
4449	SVMTRANSIENT SvmTransient;
4450	RT_ZERO(SvmTransient);
4451	SvmTransient.fUpdateTscOffsetting = true;
4452	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4453
4454	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4455	for (;;)
4456	{
4457	Assert(!HMR0SuspendPending());
4458	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4459
4460	/* Preparatory work for running nested-guest code, this may force us to return to
4461	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4462	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4463	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4464	if (rc != VINF_SUCCESS)
4465	break;
4466
4467	/*
4468	* No longjmps to ring-3 from this point on!!!
4469	*
4470	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4471	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4472	*/
4473	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4474	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4475
4476	/* Restore any residual host-state and save any bits shared between host and guest
4477	into the guest-CPU state. Re-enables interrupts! */
4478	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4479
4480	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4481	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4482	{
4483	if (rc == VINF_SUCCESS)
4484	rc = VERR_SVM_INVALID_GUEST_STATE;
4485	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4486	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4487	break;
4488	}
4489
4490	/* Handle the #VMEXIT. */
4491	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4492	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4493	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4494	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4495	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4496	if (rc != VINF_SUCCESS)
4497	break;
4498	if (++(*pcLoops) >= cMaxResumeLoops)
4499	{
4500	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4501	rc = VINF_EM_RAW_INTERRUPT;
4502	break;
4503	}
4504	}
4505
4506	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4507	return rc;
4508	}
4509
4510
4511	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4512	/**
4513	* Runs the nested-guest code using AMD-V.
4514	*
4515	* @returns Strict VBox status code.
4516	* @param pVCpu The cross context virtual CPU structure.
4517	* @param pcLoops Pointer to the number of executed loops. If we're switching
4518	* from the guest-code execution loop to this nested-guest
4519	* execution loop pass the remainder value, else pass 0.
4520	*/
4521	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4522	{
4523	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4524	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4525	Assert(pcLoops);
4526	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4527	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4528	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4529
4530	SVMTRANSIENT SvmTransient;
4531	RT_ZERO(SvmTransient);
4532	SvmTransient.fUpdateTscOffsetting = true;
4533	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4534	SvmTransient.fIsNestedGuest = true;
4535
4536	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4537	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4538
4539	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4540	for (;;)
4541	{
4542	Assert(!HMR0SuspendPending());
4543	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4544
4545	/* Preparatory work for running nested-guest code, this may force us to return to
4546	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4547	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4548	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4549	if ( rc != VINF_SUCCESS
4550	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4551	break;
4552
4553	/*
4554	* No longjmps to ring-3 from this point on!!!
4555	*
4556	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4557	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4558	*/
4559	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4560
4561	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4562
4563	/* Restore any residual host-state and save any bits shared between host and guest
4564	into the guest-CPU state. Re-enables interrupts! */
4565	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4566
4567	if (RT_LIKELY( rc == VINF_SUCCESS
4568	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4569	{ /* extremely likely */ }
4570	else
4571	{
4572	/* VMRUN failed, shouldn't really happen, Guru. */
4573	if (rc != VINF_SUCCESS)
4574	break;
4575
4576	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4577	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4578	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4579	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4580	break;
4581	}
4582
4583	/* Handle the #VMEXIT. */
4584	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4585	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4586	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4587	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4588	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4589	if (rc == VINF_SUCCESS)
4590	{
4591	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4592	{
4593	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4594	rc = VINF_SVM_VMEXIT;
4595	}
4596	else
4597	{
4598	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4599	continue;
4600	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4601	rc = VINF_EM_RAW_INTERRUPT;
4602	}
4603	}
4604	else
4605	Assert(rc != VINF_SVM_VMEXIT);
4606	break;
4607	/** @todo NSTSVM: handle single-stepping. */
4608	}
4609
4610	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4611	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4612
4613	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4614	return rc;
4615	}
4616	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4617
4618
4619	/**
4620	* Checks if any expensive dtrace probes are enabled and we should go to the
4621	* debug loop.
4622	*
4623	* @returns true if we should use debug loop, false if not.
4624	*/
4625	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4626	{
4627	/* It's probably faster to OR the raw 32-bit counter variables together.
4628	Since the variables are in an array and the probes are next to one
4629	another (more or less), we have good locality. So, better read
4630	eight-nine cache lines ever time and only have one conditional, than
4631	128+ conditionals, right? */
4632	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4633	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4634	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4635	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4636	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4637	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4638	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4639	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4640	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4641	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4642	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4643	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4644	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4645	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4646	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4647	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4648	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4649	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4650	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4651	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4652	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4653	) != 0
4654	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4655	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4656	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4657	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4658	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4659	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4660	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4661	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4662	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4663	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4664	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4665	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4666	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4673	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4675	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4676	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4677	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4678	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4679	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4680	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4681	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4682	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4683	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4684	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4685	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4686	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4687	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4689	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4690	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4691	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4692	) != 0
4693	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4694	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4695	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4696	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4697	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4698	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4699	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4700	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4701	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4702	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4703	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4704	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4705	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4713	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4715	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4716	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4717	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4718	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4719	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4720	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4721	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4722	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4723	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4724	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4725	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4726	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4727	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4729	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4730	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4731	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4732	) != 0;
4733	}
4734
4735
4736	/**
4737	* Runs the guest code using AMD-V.
4738	*
4739	* @returns Strict VBox status code.
4740	* @param pVCpu The cross context virtual CPU structure.
4741	*/
4742	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4743	{
4744	AssertPtr(pVCpu);
4745	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4746	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4747	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4748	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4749
4750	uint32_t cLoops = 0;
4751	VBOXSTRICTRC rc;
4752	for (;;)
4753	{
4754	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4755	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4756	#else
4757	NOREF(pCtx);
4758	bool const fInNestedGuestMode = false;
4759	#endif
4760	if (!fInNestedGuestMode)
4761	{
4762	if ( !pVCpu->hm.s.fUseDebugLoop
4763	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4764	&& !DBGFIsStepping(pVCpu)
4765	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4766	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4767	else
4768	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4769	}
4770	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4771	else
4772	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4773
4774	if (rc == VINF_SVM_VMRUN)
4775	{
4776	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4777	continue;
4778	}
4779	if (rc == VINF_SVM_VMEXIT)
4780	{
4781	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4782	continue;
4783	}
4784	#endif
4785	break;
4786	}
4787
4788	/* Fixup error codes. */
4789	if (rc == VERR_EM_INTERPRETER)
4790	rc = VINF_EM_RAW_EMULATE_INSTR;
4791	else if (rc == VINF_EM_RESET)
4792	rc = VINF_EM_TRIPLE_FAULT;
4793
4794	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4795	rc = hmR0SvmExitToRing3(pVCpu, rc);
4796	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4797	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4798	return rc;
4799	}
4800
4801	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4802
4803	/**
4804	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4805	*
4806	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4807	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4808	*/
4809	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4810	{
4811	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4812	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4813	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4814	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4815	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4816	const bool fRep = pIoExitInfo->n.u1Rep;
4817	const bool fStrIo = pIoExitInfo->n.u1Str;
4818
4819	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4820	NULL /* pIoExitInfo */);
4821	}
4822
4823
4824	/**
4825	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4826	* SVM_EXIT_INVALID).
4827	*
4828	* @returns VBox status code (informational status codes included).
4829	* @param pVCpu The cross context virtual CPU structure.
4830	* @param pSvmTransient Pointer to the SVM transient structure.
4831	*/
4832	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4833	{
4834	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4835	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4836	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4837
4838	/*
4839	* We import the complete state here because we use separate VMCBs for the guest and the
4840	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4841	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4842	*/
4843	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4844	do { \
4845	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4846	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4847	} while (0)
4848
4849	/*
4850	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4851	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4852	*/
4853	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4854	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4855	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4856	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4857	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4858	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4859
4860	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4861	switch (uExitCode)
4862	{
4863	case SVM_EXIT_CPUID:
4864	{
4865	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4866	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4867	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4868	}
4869
4870	case SVM_EXIT_RDTSC:
4871	{
4872	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4873	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4874	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4875	}
4876
4877	case SVM_EXIT_RDTSCP:
4878	{
4879	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4880	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4881	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4882	}
4883
4884	case SVM_EXIT_MONITOR:
4885	{
4886	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4887	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4888	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4889	}
4890
4891	case SVM_EXIT_MWAIT:
4892	{
4893	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4894	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4895	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4896	}
4897
4898	case SVM_EXIT_HLT:
4899	{
4900	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4901	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4902	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4903	}
4904
4905	case SVM_EXIT_MSR:
4906	{
4907	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4908	{
4909	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4910	uint16_t offMsrpm;
4911	uint8_t uMsrpmBit;
4912	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4913	if (RT_SUCCESS(rc))
4914	{
4915	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4916	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4917
4918	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4919	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4920	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4921
4922	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4923	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4924	{
4925	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4926	}
4927	}
4928	else
4929	{
4930	/*
4931	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4932	* See AMD-V spec. "15.11 MSR Intercepts".
4933	*/
4934	Assert(rc == VERR_OUT_OF_RANGE);
4935	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4936	}
4937	}
4938	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4939	}
4940
4941	case SVM_EXIT_IOIO:
4942	{
4943	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4944	{
4945	SVMIOIOEXITINFO IoExitInfo;
4946	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4947	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4948	if (fIntercept)
4949	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4950	}
4951	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4952	}
4953
4954	case SVM_EXIT_XCPT_PF:
4955	{
4956	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4957	if (pVM->hmr0.s.fNestedPaging)
4958	{
4959	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4960	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4961
4962	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4963	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4964	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4965
4966	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4967	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4968	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4969	return VINF_SUCCESS;
4970	}
4971	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4972	}
4973
4974	case SVM_EXIT_XCPT_UD:
4975	{
4976	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4977	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4978	hmR0SvmSetPendingXcptUD(pVCpu);
4979	return VINF_SUCCESS;
4980	}
4981
4982	case SVM_EXIT_XCPT_MF:
4983	{
4984	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4985	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4986	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4987	}
4988
4989	case SVM_EXIT_XCPT_DB:
4990	{
4991	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4992	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4993	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
4994	}
4995
4996	case SVM_EXIT_XCPT_AC:
4997	{
4998	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
4999	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5000	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5001	}
5002
5003	case SVM_EXIT_XCPT_BP:
5004	{
5005	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5006	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5007	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5008	}
5009
5010	case SVM_EXIT_READ_CR0:
5011	case SVM_EXIT_READ_CR3:
5012	case SVM_EXIT_READ_CR4:
5013	{
5014	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5015	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5016	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5017	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5018	}
5019
5020	case SVM_EXIT_CR0_SEL_WRITE:
5021	{
5022	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5023	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5024	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5025	}
5026
5027	case SVM_EXIT_WRITE_CR0:
5028	case SVM_EXIT_WRITE_CR3:
5029	case SVM_EXIT_WRITE_CR4:
5030	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5031	{
5032	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5033	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5034
5035	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5036	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5037	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5038	}
5039
5040	case SVM_EXIT_PAUSE:
5041	{
5042	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5043	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5044	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5045	}
5046
5047	case SVM_EXIT_VINTR:
5048	{
5049	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5050	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5051	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5052	}
5053
5054	case SVM_EXIT_INTR:
5055	case SVM_EXIT_NMI:
5056	case SVM_EXIT_SMI:
5057	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5058	{
5059	/*
5060	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5061	*
5062	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5063	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5064	* nested-guest #VMEXIT.
5065	*
5066	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5067	* while trying to inject interrupts, see comment at the top of this function.
5068	*/
5069	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5070	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5071	}
5072
5073	case SVM_EXIT_FERR_FREEZE:
5074	{
5075	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5076	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5077	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5078	}
5079
5080	case SVM_EXIT_INVLPG:
5081	{
5082	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5083	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5084	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5085	}
5086
5087	case SVM_EXIT_WBINVD:
5088	{
5089	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5090	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5091	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5092	}
5093
5094	case SVM_EXIT_INVD:
5095	{
5096	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5097	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5098	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5099	}
5100
5101	case SVM_EXIT_RDPMC:
5102	{
5103	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5104	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5105	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5106	}
5107
5108	default:
5109	{
5110	switch (uExitCode)
5111	{
5112	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5113	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5114	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5115	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5116	{
5117	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5118	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5119	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5120	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5121	}
5122
5123	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5124	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5125	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5126	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5127	{
5128	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5129	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5130	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5131	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5132	}
5133
5134	case SVM_EXIT_XCPT_DE:
5135	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5136	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5137	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5138	case SVM_EXIT_XCPT_OF:
5139	case SVM_EXIT_XCPT_BR:
5140	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5141	case SVM_EXIT_XCPT_NM:
5142	case SVM_EXIT_XCPT_DF:
5143	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5144	case SVM_EXIT_XCPT_TS:
5145	case SVM_EXIT_XCPT_NP:
5146	case SVM_EXIT_XCPT_SS:
5147	case SVM_EXIT_XCPT_GP:
5148	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5149	case SVM_EXIT_XCPT_15: /* Reserved. */
5150	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5151	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5152	case SVM_EXIT_XCPT_MC:
5153	case SVM_EXIT_XCPT_XF:
5154	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5155	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5156	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5157	{
5158	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5159	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5160	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5161	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5162	}
5163
5164	case SVM_EXIT_XSETBV:
5165	{
5166	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5167	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5168	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5169	}
5170
5171	case SVM_EXIT_TASK_SWITCH:
5172	{
5173	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5174	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5175	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5176	}
5177
5178	case SVM_EXIT_IRET:
5179	{
5180	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5181	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5182	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5183	}
5184
5185	case SVM_EXIT_SHUTDOWN:
5186	{
5187	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5188	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5189	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5190	}
5191
5192	case SVM_EXIT_VMMCALL:
5193	{
5194	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5195	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5196	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5197	}
5198
5199	case SVM_EXIT_CLGI:
5200	{
5201	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5202	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5203	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5204	}
5205
5206	case SVM_EXIT_STGI:
5207	{
5208	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5209	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5210	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5211	}
5212
5213	case SVM_EXIT_VMLOAD:
5214	{
5215	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5216	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5217	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5218	}
5219
5220	case SVM_EXIT_VMSAVE:
5221	{
5222	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5223	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5224	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5225	}
5226
5227	case