HMSVMR0.cpp

Last change on this file was 107854, checked in by vboxsync, 6 days ago
x86.h,VMM: More AMD CPUID bits; addressed some old todos related to these; fixed bugs in svn & vmx world switcher (sanity checks, ++). jiraref:VBP-947 bugref:10738
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 381.5 KB

Line
1	/* $Id: HMSVMR0.cpp 107854 2025-01-18 23:59:26Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_HM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/thread.h>
36
37	#include <VBox/vmm/pdmapi.h>
38	#include <VBox/vmm/dbgf.h>
39	#include <VBox/vmm/iem.h>
40	#include <VBox/vmm/iom.h>
41	#include <VBox/vmm/tm.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/gcm.h>
44	#include <VBox/vmm/gim.h>
45	#include <VBox/vmm/pdmapic.h>
46	#include "HMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include <VBox/err.h>
49	#include "HMSVMR0.h"
50	#include "dtrace/VBoxVMM.h"
51
52	#ifdef DEBUG_ramshankar
53	# define HMSVM_SYNC_FULL_GUEST_STATE
54	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
55	# define HMSVM_ALWAYS_TRAP_PF
56	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
57	#endif
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	#ifdef VBOX_WITH_STATISTICS
64	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
66	if ((u64ExitCode) == SVM_EXIT_NPF) \
67	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
68	else \
69	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
70	} while (0)
71
72	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
74	if ((u64ExitCode) == SVM_EXIT_NPF) \
75	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
76	else \
77	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
78	} while (0)
79
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
81	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
82	if ((u64ExitCode) == SVM_EXIT_NPF) \
83	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
84	else \
85	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
86	} while (0)
87	#else
88	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
89	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
90	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
91	#endif /* !VBOX_WITH_STATISTICS */
92
93	/** If we decide to use a function table approach this can be useful to
94	* switch to a "static DECLCALLBACK(int)". */
95	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
96
97	/**
98	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
99	* guest using hardware-assisted SVM.
100	*
101	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
102	* are swapped and restored across the world-switch and also registers like
103	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
104	* \#VMEXIT.
105	*/
106	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
107	\| CPUMCTX_EXTRN_RFLAGS \
108	\| CPUMCTX_EXTRN_RAX \
109	\| CPUMCTX_EXTRN_RSP \
110	\| CPUMCTX_EXTRN_SREG_MASK \
111	\| CPUMCTX_EXTRN_CR0 \
112	\| CPUMCTX_EXTRN_CR2 \
113	\| CPUMCTX_EXTRN_CR3 \
114	\| CPUMCTX_EXTRN_TABLE_MASK \
115	\| CPUMCTX_EXTRN_DR6 \
116	\| CPUMCTX_EXTRN_DR7 \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_HWVIRT \
121	\| CPUMCTX_EXTRN_INHIBIT_INT \
122	\| CPUMCTX_EXTRN_HM_SVM_MASK)
123
124	/**
125	* Subset of the guest-CPU state that is shared between the guest and host.
126	*/
127	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
128
129	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
130	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
131	do { \
132	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
133	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
134	} while (0)
135
136	/** Assert that the required state bits are fetched. */
137	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
151
152	/** Assert that we're not executing a nested-guest. */
153	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
154	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
155	#else
156	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
157	#endif
158
159	/** Assert that we're executing a nested-guest. */
160	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
161	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
162	#else
163	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
164	#endif
165
166	/** Macro for checking and returning from the using function for
167	* \#VMEXIT intercepts that maybe caused during delivering of another
168	* event in the guest. */
169	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
170	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
171	do \
172	{ \
173	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
174	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
175	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
176	else if ( rc == VINF_EM_RESET \
177	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
178	{ \
179	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
180	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
181	} \
182	else \
183	return rc; \
184	} while (0)
185	#else
186	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
187	do \
188	{ \
189	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
190	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
191	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
192	else \
193	return rc; \
194	} while (0)
195	#endif
196
197	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
198	* instruction that exited. */
199	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
200	do { \
201	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
202	(a_rc) = VINF_EM_DBG_STEPPED; \
203	} while (0)
204
205	/** Validate segment descriptor granularity bit. */
206	#ifdef VBOX_STRICT
207	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
208	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
209	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
210	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
211	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
212	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
213	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
214	#else
215	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
216	#endif
217
218	/**
219	* Exception bitmap mask for all contributory exceptions.
220	*
221	* Page fault is deliberately excluded here as it's conditional as to whether
222	* it's contributory or benign. Page faults are handled separately.
223	*/
224	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
225	\| RT_BIT(X86_XCPT_DE))
226
227	/**
228	* Mandatory/unconditional guest control intercepts.
229	*
230	* SMIs can and do happen in normal operation. We need not intercept them
231	* while executing the guest (or nested-guest).
232	*/
233	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
234	\| SVM_CTRL_INTERCEPT_NMI \
235	\| SVM_CTRL_INTERCEPT_INIT \
236	\| SVM_CTRL_INTERCEPT_RDPMC \
237	\| SVM_CTRL_INTERCEPT_CPUID \
238	\| SVM_CTRL_INTERCEPT_RSM \
239	\| SVM_CTRL_INTERCEPT_HLT \
240	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
241	\| SVM_CTRL_INTERCEPT_MSR_PROT \
242	\| SVM_CTRL_INTERCEPT_INVLPGA \
243	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
244	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
245	\| SVM_CTRL_INTERCEPT_VMRUN \
246	\| SVM_CTRL_INTERCEPT_SKINIT \
247	\| SVM_CTRL_INTERCEPT_WBINVD \
248	\| SVM_CTRL_INTERCEPT_MONITOR \
249	\| SVM_CTRL_INTERCEPT_MWAIT \
250	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
251	\| SVM_CTRL_INTERCEPT_XSETBV)
252
253	/** @name VMCB Clean Bits.
254	*
255	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
256	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
257	* memory.
258	*
259	* @{ */
260	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
261	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
262	/** I/O permission bitmap, MSR permission bitmap. */
263	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
264	/** ASID. */
265	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
266	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
267	V_INTR_VECTOR. */
268	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
269	/** Nested Paging: Nested CR3 (nCR3), PAT. */
270	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
271	/** Control registers (CR0, CR3, CR4, EFER). */
272	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
273	/** Debug registers (DR6, DR7). */
274	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
275	/** GDT, IDT limit and base. */
276	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
277	/** Segment register: CS, SS, DS, ES limit and base. */
278	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
279	/** CR2.*/
280	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
281	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
282	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
283	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
284	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
285	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
286	/** Mask of all valid VMCB Clean bits. */
287	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
288	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
289	\| HMSVM_VMCB_CLEAN_ASID \
290	\| HMSVM_VMCB_CLEAN_INT_CTRL \
291	\| HMSVM_VMCB_CLEAN_NP \
292	\| HMSVM_VMCB_CLEAN_CRX_EFER \
293	\| HMSVM_VMCB_CLEAN_DRX \
294	\| HMSVM_VMCB_CLEAN_DT \
295	\| HMSVM_VMCB_CLEAN_SEG \
296	\| HMSVM_VMCB_CLEAN_CR2 \
297	\| HMSVM_VMCB_CLEAN_LBR \
298	\| HMSVM_VMCB_CLEAN_AVIC)
299	/** @} */
300
301	/**
302	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
303	*/
304	typedef enum SVMMSREXITREAD
305	{
306	/** Reading this MSR causes a \#VMEXIT. */
307	SVMMSREXIT_INTERCEPT_READ = 0xb,
308	/** Reading this MSR does not cause a \#VMEXIT. */
309	SVMMSREXIT_PASSTHRU_READ
310	} SVMMSREXITREAD;
311
312	/**
313	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
314	*/
315	typedef enum SVMMSREXITWRITE
316	{
317	/** Writing to this MSR causes a \#VMEXIT. */
318	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a \#VMEXIT. */
320	SVMMSREXIT_PASSTHRU_WRITE
321	} SVMMSREXITWRITE;
322
323	/**
324	* SVM \#VMEXIT handler.
325	*
326	* @returns Strict VBox status code.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pSvmTransient Pointer to the SVM-transient structure.
329	*/
330	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
331
332
333	/*********************************************************************************************************************************
334	* Internal Functions *
335	*********************************************************************************************************************************/
336	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
337	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
338
339
340	/** @name \#VMEXIT handlers.
341	* @{
342	*/
343	static FNSVMEXITHANDLER hmR0SvmExitIntr;
344	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
345	static FNSVMEXITHANDLER hmR0SvmExitInvd;
346	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
347	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
348	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
349	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
350	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
351	static FNSVMEXITHANDLER hmR0SvmExitHlt;
352	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
353	static FNSVMEXITHANDLER hmR0SvmExitMwait;
354	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
355	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
356	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
357	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
358	static FNSVMEXITHANDLER hmR0SvmExitMsr;
359	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
360	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
361	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
362	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
363	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
364	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
365	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
366	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
367	static FNSVMEXITHANDLER hmR0SvmExitPause;
368	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
369	static FNSVMEXITHANDLER hmR0SvmExitIret;
370	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
371	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
372	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
373	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
374	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
375	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
376	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
377	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
378	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
379	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
380	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
381	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
382	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
383	static FNSVMEXITHANDLER hmR0SvmExitClgi;
384	static FNSVMEXITHANDLER hmR0SvmExitStgi;
385	static FNSVMEXITHANDLER hmR0SvmExitVmload;
386	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
387	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
388	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
389	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
390	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
391	#endif
392	/** @} */
393
394	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
395	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
396	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
397	#endif
398	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
399
400
401	/*********************************************************************************************************************************
402	* Global Variables *
403	*********************************************************************************************************************************/
404	/** Ring-0 memory object for the IO bitmap. */
405	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
406	/** Physical address of the IO bitmap. */
407	static RTHCPHYS g_HCPhysIOBitmap;
408	/** Pointer to the IO bitmap. */
409	static R0PTRTYPE(void *) g_pvIOBitmap;
410
411	#ifdef VBOX_STRICT
412	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
413	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
414	# define HMSVM_LOG_CS RT_BIT_32(2)
415	# define HMSVM_LOG_SS RT_BIT_32(3)
416	# define HMSVM_LOG_FS RT_BIT_32(4)
417	# define HMSVM_LOG_GS RT_BIT_32(5)
418	# define HMSVM_LOG_LBR RT_BIT_32(6)
419	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
420	\| HMSVM_LOG_CR_REGS \
421	\| HMSVM_LOG_CS \
422	\| HMSVM_LOG_SS \
423	\| HMSVM_LOG_FS \
424	\| HMSVM_LOG_GS \
425	\| HMSVM_LOG_LBR)
426
427	/**
428	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
429	*
430	* @param pVCpu The cross context virtual CPU structure.
431	* @param pVmcb Pointer to the VM control block.
432	* @param pszPrefix Log prefix.
433	* @param fFlags Log flags, see HMSVM_LOG_XXX.
434	* @param uVerbose The verbosity level, currently unused.
435	*/
436	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
437	{
438	RT_NOREF2(pVCpu, uVerbose);
439	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
440
441	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
442	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
443
444	if (fFlags & HMSVM_LOG_RBP_RSP)
445	{
446	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
447	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
448	}
449
450	if (fFlags & HMSVM_LOG_CR_REGS)
451	{
452	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
453	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
454	}
455
456	if (fFlags & HMSVM_LOG_CS)
457	{
458	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
459	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
460	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
461	}
462	if (fFlags & HMSVM_LOG_SS)
463	{
464	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
465	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
466	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
467	}
468	if (fFlags & HMSVM_LOG_FS)
469	{
470	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
471	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
472	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
473	}
474	if (fFlags & HMSVM_LOG_GS)
475	{
476	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
477	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
478	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
479	}
480
481	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
482	if (fFlags & HMSVM_LOG_LBR)
483	{
484	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
485	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
486	}
487	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
488	}
489	#endif /* VBOX_STRICT */
490
491
492	/**
493	* Sets up and activates AMD-V on the current CPU.
494	*
495	* @returns VBox status code.
496	* @param pHostCpu The HM physical-CPU structure.
497	* @param pVM The cross context VM structure. Can be
498	* NULL after a resume!
499	* @param pvCpuPage Pointer to the global CPU page.
500	* @param HCPhysCpuPage Physical address of the global CPU page.
501	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
502	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
503	* unused).
504	*/
505	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
506	PCSUPHWVIRTMSRS pHwvirtMsrs)
507	{
508	Assert(!fEnabledByHost);
509	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
510	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
511	Assert(pvCpuPage); NOREF(pvCpuPage);
512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
513
514	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
515
516	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
517	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
518
519	/*
520	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
521	*/
522	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
523	if (u64HostEfer & MSR_K6_EFER_SVME)
524	{
525	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
526	if ( pVM
527	&& pVM->hm.s.svm.fIgnoreInUseError)
528	pHostCpu->fIgnoreAMDVInUseError = true;
529
530	if (!pHostCpu->fIgnoreAMDVInUseError)
531	{
532	ASMSetFlags(fEFlags);
533	return VERR_SVM_IN_USE;
534	}
535	}
536
537	/* Turn on AMD-V in the EFER MSR. */
538	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
539
540	/* Write the physical page address where the CPU will store the host state while executing the VM. */
541	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
542
543	/* Restore interrupts. */
544	ASMSetFlags(fEFlags);
545
546	/*
547	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
548	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
549	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
550	* entirely with before executing any guest code.
551	*/
552	pHostCpu->fFlushAsidBeforeUse = true;
553
554	/*
555	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
556	*/
557	++pHostCpu->cTlbFlushes;
558
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Deactivates AMD-V on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pHostCpu The HM physical-CPU structure.
568	* @param pvCpuPage Pointer to the global CPU page.
569	* @param HCPhysCpuPage Physical address of the global CPU page.
570	*/
571	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
572	{
573	RT_NOREF1(pHostCpu);
574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
575	AssertReturn( HCPhysCpuPage
576	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
577	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
578
579	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
580	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
581
582	/* Turn off AMD-V in the EFER MSR. */
583	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
584	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
585
586	/* Invalidate host state physical address. */
587	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
588
589	/* Restore interrupts. */
590	ASMSetFlags(fEFlags);
591
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Does global AMD-V initialization (called during module initialization).
598	*
599	* @returns VBox status code.
600	*/
601	VMMR0DECL(int) SVMR0GlobalInit(void)
602	{
603	/*
604	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
605	* intercept all IO accesses, it's done once globally here instead of per-VM.
606	*/
607	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
608	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT,
609	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
610	if (RT_FAILURE(rc))
611	return rc;
612
613	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
614	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
615
616	/* Set all bits to intercept all IO accesses. */
617	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
618
619	return VINF_SUCCESS;
620	}
621
622
623	/**
624	* Does global AMD-V termination (called during module termination).
625	*/
626	VMMR0DECL(void) SVMR0GlobalTerm(void)
627	{
628	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
629	{
630	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
631	g_pvIOBitmap = NULL;
632	g_HCPhysIOBitmap = 0;
633	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
634	}
635	}
636
637
638	/**
639	* Frees any allocated per-VCPU structures for a VM.
640	*
641	* @param pVM The cross context VM structure.
642	*/
643	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
644	{
645	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
646	{
647	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
648	AssertPtr(pVCpu);
649
650	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
651	{
652	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
653	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
654	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
655	}
656
657	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
658	{
659	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
660	pVCpu->hmr0.s.svm.pVmcb = NULL;
661	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
662	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
663	}
664
665	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
666	{
667	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
668	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
669	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
670	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
671	}
672	}
673	}
674
675
676	/**
677	* Sets pfnVMRun to the best suited variant.
678	*
679	* This must be called whenever anything changes relative to the SVMR0VMRun
680	* variant selection:
681	* - pVCpu->hm.s.fLoadSaveGuestXcr0
682	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
683	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
684	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
685	* - Perhaps: CPUMCTX.fXStateMask (windows only)
686	*
687	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
688	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
689	*/
690	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
691	{
692	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
693	{
694	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit_SansSpecCtrl },
695	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit_SansSpecCtrl },
696	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit_SansSpecCtrl },
697	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit_SansSpecCtrl },
698	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit_SansSpecCtrl },
699	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit_SansSpecCtrl },
700	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit_SansSpecCtrl },
701	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit_SansSpecCtrl },
702	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit_WithSpecCtrl },
703	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit_WithSpecCtrl },
704	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit_WithSpecCtrl },
705	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit_WithSpecCtrl },
706	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit_WithSpecCtrl },
707	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit_WithSpecCtrl },
708	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit_WithSpecCtrl },
709	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit_WithSpecCtrl },
710	};
711	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
712	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
713	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0)
714	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_SPEC_CTRL ? 8 : 0);
715	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
716	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
717	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
718	}
719
720
721	/**
722	* Selector FNHMSVMVMRUN implementation.
723	*/
724	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
725	{
726	hmR0SvmUpdateVmRunFunction(pVCpu);
727	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
728	}
729
730
731	/**
732	* Does per-VM AMD-V initialization.
733	*
734	* @returns VBox status code.
735	* @param pVM The cross context VM structure.
736	*/
737	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
738	{
739	int rc = VERR_INTERNAL_ERROR_5;
740
741	/*
742	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
743	*/
744	uint32_t u32Family;
745	uint32_t u32Model;
746	uint32_t u32Stepping;
747	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
748	{
749	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
750	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
751	}
752
753	/*
754	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
755	*/
756	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
757	{
758	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
759	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
760	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
761	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
762	}
763
764	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
765	{
766	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
767
768	/*
769	* Initialize the hardware-assisted SVM guest-execution handler.
770	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
771	*/
772	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
773
774	/*
775	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
776	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
777	*/
778	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
779	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
780	if (RT_FAILURE(rc))
781	goto failure_cleanup;
782
783	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
784	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
785	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
786
787	/*
788	* Allocate one page for the guest-state VMCB.
789	*/
790	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
791	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
792	if (RT_FAILURE(rc))
793	goto failure_cleanup;
794
795	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
796	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
797	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
798
799	/*
800	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
801	* SVM to not require one.
802	*/
803	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
804	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
805	if (RT_FAILURE(rc))
806	goto failure_cleanup;
807
808	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
809	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
810	/* Set all bits to intercept all MSR accesses (changed later on). */
811	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
812	}
813
814	return VINF_SUCCESS;
815
816	failure_cleanup:
817	hmR0SvmFreeStructs(pVM);
818	return rc;
819	}
820
821
822	/**
823	* Does per-VM AMD-V termination.
824	*
825	* @returns VBox status code.
826	* @param pVM The cross context VM structure.
827	*/
828	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
829	{
830	hmR0SvmFreeStructs(pVM);
831	return VINF_SUCCESS;
832	}
833
834
835	/**
836	* Returns whether the VMCB Clean Bits feature is supported.
837	*
838	* @returns @c true if supported, @c false otherwise.
839	* @param pVCpu The cross context virtual CPU structure.
840	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
841	*/
842	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
843	{
844	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
845	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
846	if (!fIsNestedGuest)
847	return fHostVmcbCleanBits;
848	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
849	}
850
851
852	/**
853	* Returns whether the decode assists feature is supported.
854	*
855	* @returns @c true if supported, @c false otherwise.
856	* @param pVCpu The cross context virtual CPU structure.
857	*/
858	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
859	{
860	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
861	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
862	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
863	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
864	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
865	#endif
866	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
867	}
868
869
870	/**
871	* Returns whether the NRIP_SAVE feature is supported.
872	*
873	* @returns @c true if supported, @c false otherwise.
874	* @param pVCpu The cross context virtual CPU structure.
875	*/
876	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
877	{
878	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
879	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
880	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
881	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
882	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
883	#endif
884	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
885	}
886
887
888	/**
889	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
890	*
891	* @param pVCpu The cross context virtual CPU structure.
892	* @param pbMsrBitmap Pointer to the MSR bitmap.
893	* @param idMsr The MSR for which the permissions are being set.
894	* @param enmRead MSR read permissions.
895	* @param enmWrite MSR write permissions.
896	*
897	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
898	* caller needs to take care of this.
899	*/
900	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
901	SVMMSREXITWRITE enmWrite)
902	{
903	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
904	uint16_t offMsrpm;
905	uint8_t uMsrpmBit;
906	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
907	AssertRC(rc);
908
909	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
910	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
911
912	pbMsrBitmap += offMsrpm;
913	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
914	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
915	else
916	{
917	if (!fInNestedGuestMode)
918	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
919	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
920	else
921	{
922	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
923	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
924	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
925	else
926	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
927	}
928	#endif
929	}
930
931	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
932	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
933	else
934	{
935	if (!fInNestedGuestMode)
936	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
937	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
938	else
939	{
940	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
941	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
942	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
943	else
944	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
945	}
946	#endif
947	}
948	}
949
950
951	/**
952	* Sets up AMD-V for the specified VM.
953	* This function is only called once per-VM during initalization.
954	*
955	* @returns VBox status code.
956	* @param pVM The cross context VM structure.
957	*/
958	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
959	{
960	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
961	AssertReturn(pVM, VERR_INVALID_PARAMETER);
962
963	/*
964	* Validate and copy over some parameters.
965	*/
966	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
967	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
968	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
969	pVM->hmr0.s.fNestedPaging = fNestedPaging;
970	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
971
972	/*
973	* Determin some configuration parameters.
974	*/
975	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
976	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
977	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
978
979	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
980	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
981
982	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
983	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
984	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
985
986	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
987	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
988	#endif
989
990	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
991	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
992	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
993	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
994
995	/* Always trap #AC for reasons of security. */
996	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
997
998	/* Always trap #DB for reasons of security. */
999	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
1000
1001	/* Trap exceptions unconditionally (debug purposes). */
1002	#ifdef HMSVM_ALWAYS_TRAP_PF
1003	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
1004	#endif
1005	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1006	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
1007	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
1008	\| RT_BIT_32(X86_XCPT_DE)
1009	\| RT_BIT_32(X86_XCPT_NM)
1010	\| RT_BIT_32(X86_XCPT_UD)
1011	\| RT_BIT_32(X86_XCPT_NP)
1012	\| RT_BIT_32(X86_XCPT_SS)
1013	\| RT_BIT_32(X86_XCPT_GP)
1014	\| RT_BIT_32(X86_XCPT_PF)
1015	\| RT_BIT_32(X86_XCPT_MF)
1016	;
1017	#endif
1018
1019	/* Apply the exceptions intercepts needed by the GIM provider. */
1020	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1021	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1022
1023	/* Apply the exceptions intercepts needed by the GCM fixers. */
1024	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1025	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1026
1027	/* The mesa 3d driver hack needs #GP. */
1028	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1029	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1030
1031	/* Set up unconditional intercepts and conditions. */
1032	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1033	\| SVM_CTRL_INTERCEPT_VMMCALL
1034	\| SVM_CTRL_INTERCEPT_VMSAVE
1035	\| SVM_CTRL_INTERCEPT_VMLOAD
1036	\| SVM_CTRL_INTERCEPT_CLGI
1037	\| SVM_CTRL_INTERCEPT_STGI;
1038
1039	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1040	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1041	#endif
1042
1043	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1044	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1045	{
1046	/* Virtualized VMSAVE/VMLOAD. */
1047	if (fUseVirtVmsaveVmload)
1048	{
1049	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1050	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1051	\| SVM_CTRL_INTERCEPT_VMLOAD);
1052	}
1053	else
1054	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1055
1056	/* Virtual GIF. */
1057	if (fUseVGif)
1058	{
1059	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1060	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1061	\| SVM_CTRL_INTERCEPT_STGI);
1062	}
1063	else
1064	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1065	}
1066	else
1067	#endif
1068	{
1069	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1070	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1071	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1072	}
1073
1074	/* CR4 writes must always be intercepted for tracking PGM mode changes and
1075	AVX (for XCR0 syncing during worlds switching). */
1076	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1077
1078	/* Intercept all DRx reads and writes by default. Changed later on. */
1079	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1080	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1081
1082	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1083	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1084
1085	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1086	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1087	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1088
1089	/* Set the IO permission bitmap physical addresses. */
1090	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1091
1092	/* LBR virtualization. */
1093	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1094
1095	/* The host ASID MBZ, for the guest start with 1. */
1096	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1097
1098	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1099	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1100
1101	/* Without Nested Paging, we need additionally intercepts. */
1102	if (!fNestedPaging)
1103	{
1104	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1105	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1106	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1107
1108	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1109	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1110	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1111
1112	/* Page faults must be intercepted to implement shadow paging. */
1113	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1114	}
1115
1116	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1117	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1118	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1119
1120	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1121	if (fUsePauseFilter)
1122	{
1123	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1124	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1125	if (fPauseFilterThreshold)
1126	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1127	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1128	}
1129
1130	/*
1131	* Setup the MSR permission bitmap.
1132	* The following MSRs are saved/restored automatically during the world-switch.
1133	* Don't intercept guest read/write accesses to these MSRs.
1134	*/
1135	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1136	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1137	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1138	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1139	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1140	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1141	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1142	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1143	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1144	{
1145	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1146	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1147	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1148	}
1149	else
1150	{
1151	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1152	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1153	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1154	}
1155	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1156
1157	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1158	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1159
1160	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1161	{
1162	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1163	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1164	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1165	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1166
1167	/* Copy the VMCB control area. */
1168	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1169
1170	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1171	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1172	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1173	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1174
1175	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1176	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1177
1178	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1179	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1180	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1181	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1182	}
1183
1184	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1185	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1186	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1187	#else
1188	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1189	#endif
1190	return VINF_SUCCESS;
1191	}
1192
1193
1194	/**
1195	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1196	*
1197	* @returns Pointer to the current context VMCB.
1198	* @param pVCpu The cross context virtual CPU structure.
1199	*/
1200	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1201	{
1202	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1203	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1204	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1205	#endif
1206	return pVCpu->hmr0.s.svm.pVmcb;
1207	}
1208
1209
1210	/**
1211	* Gets a pointer to the nested-guest VMCB cache.
1212	*
1213	* @returns Pointer to the nested-guest VMCB cache.
1214	* @param pVCpu The cross context virtual CPU structure.
1215	*/
1216	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1217	{
1218	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1219	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1220	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1221	#else
1222	RT_NOREF(pVCpu);
1223	return NULL;
1224	#endif
1225	}
1226
1227
1228	/**
1229	* Invalidates a guest page by guest virtual address.
1230	*
1231	* @returns VBox status code.
1232	* @param pVCpu The cross context virtual CPU structure.
1233	* @param GCVirt Guest virtual address of the page to invalidate.
1234	*/
1235	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1236	{
1237	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1238
1239	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1240
1241	/* Skip it if a TLB flush is already pending. */
1242	if (!fFlushPending)
1243	{
1244	Log4Func(("%#RGv\n", GCVirt));
1245
1246	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1247	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1248
1249	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1250	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1251	}
1252	return VINF_SUCCESS;
1253	}
1254
1255
1256	/**
1257	* Flushes the appropriate tagged-TLB entries.
1258	*
1259	* @param pHostCpu The HM physical-CPU structure.
1260	* @param pVCpu The cross context virtual CPU structure.
1261	* @param pVmcb Pointer to the VM control block.
1262	*/
1263	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1264	{
1265	/*
1266	* Force a TLB flush for the first world switch if the current CPU differs from the one
1267	* we ran on last. This can happen both for start & resume due to long jumps back to
1268	* ring-3.
1269	*
1270	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1271	* correlation between it and the physical CPU.
1272	*
1273	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1274	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1275	*/
1276	bool fNewAsid = false;
1277	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1278	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1279	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1280	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1281	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1282	#endif
1283	)
1284	{
1285	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1286	pVCpu->hmr0.s.fForceTLBFlush = true;
1287	fNewAsid = true;
1288	}
1289
1290	/* Set TLB flush state as checked until we return from the world switch. */
1291	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1292
1293	/* Check for explicit TLB flushes. */
1294	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1295	{
1296	pVCpu->hmr0.s.fForceTLBFlush = true;
1297	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1298	}
1299
1300	/*
1301	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1302	* This Host CPU requirement takes precedence.
1303	*/
1304	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1305	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1306	{
1307	pHostCpu->uCurrentAsid = 1;
1308	pVCpu->hmr0.s.uCurrentAsid = 1;
1309	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1310	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1311	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1312
1313	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1314	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1315	}
1316	else
1317	{
1318	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1319	if (pVCpu->hmr0.s.fForceTLBFlush)
1320	{
1321	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1322	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1323
1324	if (fNewAsid)
1325	{
1326	++pHostCpu->uCurrentAsid;
1327
1328	bool fHitASIDLimit = false;
1329	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1330	{
1331	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1332	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1333	fHitASIDLimit = true;
1334	}
1335
1336	if ( fHitASIDLimit
1337	\|\| pHostCpu->fFlushAsidBeforeUse)
1338	{
1339	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1340	pHostCpu->fFlushAsidBeforeUse = false;
1341	}
1342
1343	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1344	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1345	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1346	}
1347	else
1348	{
1349	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1350	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1351	else
1352	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1353	}
1354
1355	pVCpu->hmr0.s.fForceTLBFlush = false;
1356	}
1357	}
1358
1359	/* Update VMCB with the ASID. */
1360	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1361	{
1362	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1363	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1364	}
1365
1366	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1367	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1368	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1369	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1370	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1371	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1372	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1373	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1374
1375	#ifdef VBOX_WITH_STATISTICS
1376	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1377	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1378	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1379	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1380	{
1381	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1382	}
1383	else
1384	{
1385	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1386	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1387	}
1388	#endif
1389	}
1390
1391
1392	/**
1393	* Sets an exception intercept in the specified VMCB.
1394	*
1395	* @param pVmcb Pointer to the VM control block.
1396	* @param uXcpt The exception (X86_XCPT_*).
1397	*/
1398	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1399	{
1400	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1401	{
1402	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1403	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1404	}
1405	}
1406
1407
1408	/**
1409	* Clears an exception intercept in the specified VMCB.
1410	*
1411	* @param pVCpu The cross context virtual CPU structure.
1412	* @param pVmcb Pointer to the VM control block.
1413	* @param uXcpt The exception (X86_XCPT_*).
1414	*
1415	* @remarks This takes into account if we're executing a nested-guest and only
1416	* removes the exception intercept if both the guest -and- nested-guest
1417	* are not intercepting it.
1418	*/
1419	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1420	{
1421	Assert(uXcpt != X86_XCPT_DB);
1422	Assert(uXcpt != X86_XCPT_AC);
1423	Assert(uXcpt != X86_XCPT_GP);
1424	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1425	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1426	{
1427	bool fRemove = true;
1428	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1429	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1430	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1431	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1432	{
1433	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1434	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1435	}
1436	# else
1437	RT_NOREF(pVCpu);
1438	# endif
1439	if (fRemove)
1440	{
1441	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1442	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1443	}
1444	}
1445	#else
1446	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1447	#endif
1448	}
1449
1450
1451	/**
1452	* Sets a control intercept in the specified VMCB.
1453	*
1454	* @param pVmcb Pointer to the VM control block.
1455	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1456	*/
1457	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1458	{
1459	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1460	{
1461	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1462	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1463	}
1464	}
1465
1466
1467	/**
1468	* Clears a control intercept in the specified VMCB.
1469	*
1470	* @returns @c true if the intercept is still set, @c false otherwise.
1471	* @param pVCpu The cross context virtual CPU structure.
1472	* @param pVmcb Pointer to the VM control block.
1473	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1474	*
1475	* @remarks This takes into account if we're executing a nested-guest and only
1476	* removes the control intercept if both the guest -and- nested-guest
1477	* are not intercepting it.
1478	*/
1479	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1480	{
1481	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1482	{
1483	bool fRemove = true;
1484	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1485	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1486	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1487	{
1488	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1489	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1490	}
1491	#else
1492	RT_NOREF(pVCpu);
1493	#endif
1494	if (fRemove)
1495	{
1496	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1497	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1498	}
1499	}
1500
1501	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1502	}
1503
1504
1505	/**
1506	* Exports the guest (or nested-guest) CR0 into the VMCB.
1507	*
1508	* @param pVCpu The cross context virtual CPU structure.
1509	* @param pVmcb Pointer to the VM control block.
1510	*
1511	* @remarks This assumes we always pre-load the guest FPU.
1512	* @remarks No-long-jump zone!!!
1513	*/
1514	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1515	{
1516	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1517
1518	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1519	uint64_t const uGuestCr0 = pCtx->cr0;
1520	uint64_t uShadowCr0 = uGuestCr0;
1521
1522	/* Always enable caching. */
1523	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1524
1525	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1526	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1527	{
1528	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1529	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1530	}
1531
1532	/*
1533	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1534	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1535	* see @bugref{7243#c103}.
1536	*/
1537	if (!(uGuestCr0 & X86_CR0_NE))
1538	{
1539	uShadowCr0 \|= X86_CR0_NE;
1540	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1541	}
1542	else
1543	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1544
1545	/*
1546	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1547	*
1548	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1549	* see @bugref{6944}.
1550	*
1551	* We also don't ever want to honor weird things like cache disable from the guest.
1552	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1553	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1554	*/
1555	if (uShadowCr0 == uGuestCr0)
1556	{
1557	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1558	{
1559	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1560	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1561	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1562	}
1563	else
1564	{
1565	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1566	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1567	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1568	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1569	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1570	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1571	}
1572	}
1573	else
1574	{
1575	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1576	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1577	}
1578	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1579
1580	Assert(!RT_HI_U32(uShadowCr0));
1581	if (pVmcb->guest.u64CR0 != uShadowCr0)
1582	{
1583	pVmcb->guest.u64CR0 = uShadowCr0;
1584	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1585	}
1586	}
1587
1588
1589	/**
1590	* Exports the guest (or nested-guest) CR3 into the VMCB.
1591	*
1592	* @param pVCpu The cross context virtual CPU structure.
1593	* @param pVmcb Pointer to the VM control block.
1594	*
1595	* @remarks No-long-jump zone!!!
1596	*/
1597	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1598	{
1599	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1600
1601	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1602	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1603	if (pVM->hmr0.s.fNestedPaging)
1604	{
1605	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1606	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1607	pVmcb->guest.u64CR3 = pCtx->cr3;
1608	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1609	}
1610	else
1611	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1612
1613	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1614	}
1615
1616
1617	/**
1618	* Exports the guest (or nested-guest) CR4 into the VMCB.
1619	*
1620	* @param pVCpu The cross context virtual CPU structure.
1621	* @param pVmcb Pointer to the VM control block.
1622	*
1623	* @remarks No-long-jump zone!!!
1624	*/
1625	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1626	{
1627	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1628
1629	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1630	uint64_t uShadowCr4 = pCtx->cr4;
1631	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1632	{
1633	switch (pVCpu->hm.s.enmShadowMode)
1634	{
1635	case PGMMODE_REAL:
1636	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1637	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1638
1639	case PGMMODE_32_BIT: /* 32-bit paging. */
1640	uShadowCr4 &= ~X86_CR4_PAE;
1641	break;
1642
1643	case PGMMODE_PAE: /* PAE paging. */
1644	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1645	/** Must use PAE paging as we could use physical memory > 4 GB */
1646	uShadowCr4 \|= X86_CR4_PAE;
1647	break;
1648
1649	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1650	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1651	#ifdef VBOX_WITH_64_BITS_GUESTS
1652	break;
1653	#else
1654	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1655	#endif
1656
1657	default: /* shut up gcc */
1658	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1659	}
1660	}
1661
1662	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1663	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1664	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1665	{
1666	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1667	hmR0SvmUpdateVmRunFunction(pVCpu);
1668	}
1669
1670	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1671	if (uShadowCr4 == pCtx->cr4)
1672	{
1673	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1674	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1675	else
1676	{
1677	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1678	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1679	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1680	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1681	}
1682	}
1683	else
1684	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1685
1686	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking
1687	PGM mode changes and AVX (for XCR0 syncing during worlds switching). */
1688	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1689
1690	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1691	Assert(!RT_HI_U32(uShadowCr4));
1692	pVmcb->guest.u64CR4 = uShadowCr4;
1693	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1694
1695	return VINF_SUCCESS;
1696	}
1697
1698
1699	/**
1700	* Exports the guest (or nested-guest) control registers into the VMCB.
1701	*
1702	* @returns VBox status code.
1703	* @param pVCpu The cross context virtual CPU structure.
1704	* @param pVmcb Pointer to the VM control block.
1705	*
1706	* @remarks No-long-jump zone!!!
1707	*/
1708	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1709	{
1710	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1713	{
1714	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1715	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1716
1717	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1718	{
1719	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1720	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1721	}
1722
1723	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1724	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1725
1726	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1727	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1728	{
1729	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1730	if (RT_FAILURE(rc))
1731	return rc;
1732	}
1733
1734	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1735	}
1736	return VINF_SUCCESS;
1737	}
1738
1739
1740	/**
1741	* Exports the guest (or nested-guest) segment registers into the VMCB.
1742	*
1743	* @param pVCpu The cross context virtual CPU structure.
1744	* @param pVmcb Pointer to the VM control block.
1745	*
1746	* @remarks No-long-jump zone!!!
1747	*/
1748	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1749	{
1750	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1751	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1752
1753	/* Guest segment registers. */
1754	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1755	{
1756	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1757	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1758
1759	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1760	{
1761	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1762	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1763	}
1764
1765	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1766	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1767
1768	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1769	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1770
1771	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1772	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1773
1774	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1775	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1776
1777	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1778	}
1779
1780	/* Guest TR. */
1781	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1782	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1783
1784	/* Guest LDTR. */
1785	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1786	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1787
1788	/* Guest GDTR. */
1789	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1790	{
1791	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1792	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1793	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1794	}
1795
1796	/* Guest IDTR. */
1797	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1798	{
1799	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1800	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1801	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1802	}
1803
1804	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1805	\| HM_CHANGED_GUEST_TABLE_MASK);
1806	}
1807
1808
1809	/**
1810	* Exports the guest (or nested-guest) MSRs into the VMCB.
1811	*
1812	* @param pVCpu The cross context virtual CPU structure.
1813	* @param pVmcb Pointer to the VM control block.
1814	*
1815	* @remarks No-long-jump zone!!!
1816	*/
1817	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1818	{
1819	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1820	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1821
1822	/* Guest Sysenter MSRs. */
1823	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1824	{
1825	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1826	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1827
1828	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1829	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1830
1831	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1832	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1833	}
1834
1835	/*
1836	* Guest EFER MSR.
1837	* AMD-V requires guest EFER.SVME to be set. Weird.
1838	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1839	*/
1840	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1841	{
1842	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1843	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1844	}
1845
1846	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1847	if ( !CPUMIsGuestInLongModeEx(pCtx)
1848	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1849	{
1850	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1851	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1852	}
1853
1854	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1855	{
1856	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1857	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1858	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1859	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1860	}
1861
1862	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1863	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1864
1865	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1866	\| HM_CHANGED_GUEST_EFER_MSR
1867	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1868	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1869
1870	/*
1871	* Setup the PAT MSR (applicable for Nested Paging only).
1872	*
1873	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1874	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1875	*
1876	* While guests can modify and see the modified values through the shadow values,
1877	* we shall not honor any guest modifications of this MSR to ensure caching is always
1878	* enabled similar to how we clear CR0.CD and NW bits.
1879	*
1880	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1881	*/
1882	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1883
1884	/* Enable the last branch record bit if LBR virtualization is enabled. */
1885	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1886	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1887	}
1888
1889
1890	/**
1891	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1892	* the necessary intercepts accordingly.
1893	*
1894	* @param pVCpu The cross context virtual CPU structure.
1895	* @param pVmcb Pointer to the VM control block.
1896	*
1897	* @remarks No-long-jump zone!!!
1898	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1899	*/
1900	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1901	{
1902	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1903
1904	/** @todo Figure out stepping with nested-guest. */
1905	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1906	{
1907	/*
1908	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1909	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1910	* Instead, they are strictly only requested when the nested hypervisor intercepts
1911	* them -- handled while merging VMCB controls.
1912	*
1913	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1914	* then the nested-guest debug state should be actively loaded on the host so that
1915	* nested-guest reads/writes its own debug registers without causing VM-exits.
1916	*/
1917	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1918	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1919	&& !CPUMIsGuestDebugStateActive(pVCpu))
1920	{
1921	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1922	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1923	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1924	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1925	}
1926
1927	pVmcb->guest.u64DR6 = pCtx->dr[6];
1928	pVmcb->guest.u64DR7 = pCtx->dr[7];
1929	return;
1930	}
1931
1932	/*
1933	* Anyone single stepping on the host side? If so, we'll have to use the
1934	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1935	* the VMM level like the VT-x implementations does.
1936	*/
1937	bool fInterceptMovDRx = false;
1938	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1939	if (fStepping)
1940	{
1941	pVCpu->hmr0.s.fClearTrapFlag = true;
1942	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1943	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1944	}
1945
1946	if ( fStepping
1947	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1948	{
1949	/*
1950	* Use the combined guest and host DRx values found in the hypervisor
1951	* register set because the debugger has breakpoints active or someone
1952	* is single stepping on the host side.
1953	*
1954	* Note! DBGF expects a clean DR6 state before executing guest code.
1955	*/
1956	if (!CPUMIsHyperDebugStateActive(pVCpu))
1957	{
1958	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1959	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1960	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1961	}
1962
1963	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1964	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1965	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1966	{
1967	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1968	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1969	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1970	}
1971
1972	/** @todo If we cared, we could optimize to allow the guest to read registers
1973	* with the same values. */
1974	fInterceptMovDRx = true;
1975	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1976	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1977	}
1978	else
1979	{
1980	/*
1981	* Update DR6, DR7 with the guest values if necessary.
1982	*/
1983	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1984	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1985	{
1986	pVmcb->guest.u64DR7 = pCtx->dr[7];
1987	pVmcb->guest.u64DR6 = pCtx->dr[6];
1988	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1989	}
1990	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1991
1992	/*
1993	* If the guest has enabled debug registers, we need to load them prior to
1994	* executing guest code so they'll trigger at the right time.
1995	*/
1996	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1997	{
1998	if (!CPUMIsGuestDebugStateActive(pVCpu))
1999	{
2000	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
2001	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2002	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2003	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2004	}
2005	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2006	}
2007	/*
2008	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2009	* intercept #DB as DR6 is updated in the VMCB.
2010	*
2011	* Note! If we cared and dared, we could skip intercepting \#DB here.
2012	* However, \#DB shouldn't be performance critical, so we'll play safe
2013	* and keep the code similar to the VT-x code and always intercept it.
2014	*/
2015	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2016	fInterceptMovDRx = true;
2017	}
2018
2019	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2020	if (fInterceptMovDRx)
2021	{
2022	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2023	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2024	{
2025	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2026	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2027	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2028	}
2029	}
2030	else
2031	{
2032	if ( pVmcb->ctrl.u16InterceptRdDRx
2033	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2034	{
2035	pVmcb->ctrl.u16InterceptRdDRx = 0;
2036	pVmcb->ctrl.u16InterceptWrDRx = 0;
2037	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2038	}
2039	}
2040	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2041	}
2042
2043	/**
2044	* Exports the hardware virtualization state into the nested-guest
2045	* VMCB.
2046	*
2047	* @param pVCpu The cross context virtual CPU structure.
2048	* @param pVmcb Pointer to the VM control block.
2049	*
2050	* @remarks No-long-jump zone!!!
2051	*/
2052	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2053	{
2054	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2055
2056	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2057	{
2058	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2059	{
2060	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2061	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2062
2063	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2064	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2065	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2066	NOREF(pVM);
2067
2068	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2069	}
2070
2071	/*
2072	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2073	* since SVM doesn't have a preemption timer.
2074	*
2075	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2076	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2077	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2078	*/
2079	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2080	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2081	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2082	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2083	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2084	{
2085	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2086	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2087	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2088	}
2089	else
2090	{
2091	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2092	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2093	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2094	}
2095	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2096
2097	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2098	}
2099	}
2100
2101
2102	/**
2103	* Exports the guest APIC TPR state into the VMCB.
2104	*
2105	* @returns VBox status code.
2106	* @param pVCpu The cross context virtual CPU structure.
2107	* @param pVmcb Pointer to the VM control block.
2108	*/
2109	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2110	{
2111	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2112
2113	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2114	{
2115	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2116	if ( PDMHasApic(pVM)
2117	&& PDMApicIsEnabled(pVCpu))
2118	{
2119	bool fPendingIntr;
2120	uint8_t u8Tpr;
2121	int rc = PDMApicGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2122	AssertRCReturn(rc, rc);
2123
2124	/* Assume that we need to trap all TPR accesses and thus need not check on
2125	every #VMEXIT if we should update the TPR. */
2126	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2127	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2128
2129	if (!pVM->hm.s.fTprPatchingActive)
2130	{
2131	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2132	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2133
2134	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2135	can deliver the interrupt to the guest. */
2136	if (fPendingIntr)
2137	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2138	else
2139	{
2140	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2141	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2142	}
2143
2144	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2145	}
2146	else
2147	{
2148	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2149	pVmcb->guest.u64LSTAR = u8Tpr;
2150	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2151
2152	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2153	if (fPendingIntr)
2154	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2155	else
2156	{
2157	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2158	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2159	}
2160	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2161	}
2162	}
2163	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2164	}
2165	return VINF_SUCCESS;
2166	}
2167
2168
2169	/**
2170	* Sets up the exception interrupts required for guest execution in the VMCB.
2171	*
2172	* @param pVCpu The cross context virtual CPU structure.
2173	* @param pVmcb Pointer to the VM control block.
2174	*
2175	* @remarks No-long-jump zone!!!
2176	*/
2177	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2178	{
2179	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2180
2181	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2182	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2183	{
2184	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2185	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2186	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2187	else
2188	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2189
2190	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2191	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledSwBreakpoints)
2192	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2193	else
2194	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2195
2196	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2197	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2198	}
2199	}
2200
2201
2202	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2203	/**
2204	* Merges guest and nested-guest intercepts for executing the nested-guest using
2205	* hardware-assisted SVM.
2206	*
2207	* This merges the guest and nested-guest intercepts in a way that if the outer
2208	* guest intercept is set we need to intercept it in the nested-guest as
2209	* well.
2210	*
2211	* @param pVCpu The cross context virtual CPU structure.
2212	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2213	*/
2214	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2215	{
2216	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2217	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2218	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2219	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2220
2221	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2222	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2223	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2224
2225	/* Always intercept CR4 writes for tracking PGM mode changes and AVX (for
2226	XCR0 syncing during worlds switching). */
2227	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2228
2229	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2230	if (!pVM->hmr0.s.fNestedPaging)
2231	{
2232	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2233	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2234	}
2235
2236	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2237	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2238	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2239
2240	/*
2241	* Merge the guest's exception intercepts into the nested-guest VMCB.
2242	*
2243	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2244	* while executing the nested-guest.
2245	*
2246	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2247	* be tweaked later depending on how we wish to implement breakpoints.
2248	*
2249	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2250	* loaded into their guests, not ours.
2251	*
2252	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2253	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2254	*/
2255	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2256	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2257	& ~( RT_BIT(X86_XCPT_UD)
2258	\| RT_BIT(X86_XCPT_BP)
2259	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2260	#else
2261	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2262	#endif
2263
2264	/*
2265	* Adjust intercepts while executing the nested-guest that differ from the
2266	* outer guest intercepts.
2267	*
2268	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2269	* that belong to the nested-guest to the outer guest.
2270	*
2271	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2272	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2273	*/
2274	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2275	\| SVM_CTRL_INTERCEPT_VMMCALL))
2276	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2277
2278	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2279	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2280
2281	/* Finally, update the VMCB clean bits. */
2282	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2283	}
2284	#endif
2285
2286
2287	/**
2288	* Enters the AMD-V session.
2289	*
2290	* @returns VBox status code.
2291	* @param pVCpu The cross context virtual CPU structure.
2292	*/
2293	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2294	{
2295	AssertPtr(pVCpu);
2296	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2297	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2298
2299	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2300	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2301	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2302
2303	pVCpu->hmr0.s.fLeaveDone = false;
2304	return VINF_SUCCESS;
2305	}
2306
2307
2308	/**
2309	* Thread-context callback for AMD-V.
2310	*
2311	* This is used together with RTThreadCtxHookCreate() on platforms which
2312	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2313	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2314	*
2315	* @param enmEvent The thread-context event.
2316	* @param pVCpu The cross context virtual CPU structure.
2317	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2318	* @thread EMT(pVCpu)
2319	*/
2320	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2321	{
2322	NOREF(fGlobalInit);
2323
2324	switch (enmEvent)
2325	{
2326	case RTTHREADCTXEVENT_OUT:
2327	{
2328	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2329	VMCPU_ASSERT_EMT(pVCpu);
2330
2331	/* No longjmps (log-flush, locks) in this fragile context. */
2332	VMMRZCallRing3Disable(pVCpu);
2333
2334	if (!pVCpu->hmr0.s.fLeaveDone)
2335	{
2336	hmR0SvmLeave(pVCpu, false /* fImportState */);
2337	pVCpu->hmr0.s.fLeaveDone = true;
2338	}
2339
2340	/* Leave HM context, takes care of local init (term). */
2341	int rc = HMR0LeaveCpu(pVCpu);
2342	AssertRC(rc); NOREF(rc);
2343
2344	/* Restore longjmp state. */
2345	VMMRZCallRing3Enable(pVCpu);
2346	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2347	break;
2348	}
2349
2350	case RTTHREADCTXEVENT_IN:
2351	{
2352	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2353	VMCPU_ASSERT_EMT(pVCpu);
2354
2355	/* No longjmps (log-flush, locks) in this fragile context. */
2356	VMMRZCallRing3Disable(pVCpu);
2357
2358	/*
2359	* Initialize the bare minimum state required for HM. This takes care of
2360	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2361	*/
2362	int rc = hmR0EnterCpu(pVCpu);
2363	AssertRC(rc); NOREF(rc);
2364	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2365	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2366
2367	pVCpu->hmr0.s.fLeaveDone = false;
2368
2369	/* Restore longjmp state. */
2370	VMMRZCallRing3Enable(pVCpu);
2371	break;
2372	}
2373
2374	default:
2375	break;
2376	}
2377	}
2378
2379
2380	/**
2381	* Saves the host state.
2382	*
2383	* @returns VBox status code.
2384	* @param pVCpu The cross context virtual CPU structure.
2385	*
2386	* @remarks No-long-jump zone!!!
2387	*/
2388	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2389	{
2390	NOREF(pVCpu);
2391
2392	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2393	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2394	return VINF_SUCCESS;
2395	}
2396
2397
2398	/**
2399	* Exports the guest or nested-guest state from the virtual-CPU context into the
2400	* VMCB.
2401	*
2402	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2403	* code based on the virtual-CPU mode.
2404	*
2405	* @returns VBox status code.
2406	* @param pVCpu The cross context virtual CPU structure.
2407	* @param pSvmTransient Pointer to the SVM-transient structure.
2408	*
2409	* @remarks No-long-jump zone!!!
2410	*/
2411	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2412	{
2413	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2414
2415	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2416	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2417	Assert(pVmcb);
2418
2419	pVmcb->guest.u64RIP = pCtx->rip;
2420	pVmcb->guest.u64RSP = pCtx->rsp;
2421	pVmcb->guest.u64RFlags = pCtx->eflags.u;
2422	pVmcb->guest.u64RAX = pCtx->rax;
2423
2424	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2425	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2426
2427	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2428	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2429	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2430	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2431	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2432
2433	ASMSetFlags(fEFlags);
2434
2435	if (!fIsNestedGuest)
2436	{
2437	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2438	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2439	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2440	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2441	}
2442
2443	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2444	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2445	\| HM_CHANGED_GUEST_RFLAGS
2446	\| HM_CHANGED_GUEST_GPRS_MASK
2447	\| HM_CHANGED_GUEST_X87
2448	\| HM_CHANGED_GUEST_SSE_AVX
2449	\| HM_CHANGED_GUEST_OTHER_XSAVE
2450	\| HM_CHANGED_GUEST_XCRx
2451	\| HM_CHANGED_GUEST_TSC_AUX
2452	\| HM_CHANGED_GUEST_OTHER_MSRS;
2453	if (fIsNestedGuest)
2454	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2455	\| HM_CHANGED_GUEST_APIC_TPR;
2456
2457	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2458	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2459
2460	#ifdef VBOX_STRICT
2461	/*
2462	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2463	* except for the host-context and/or shared host-guest context bits.
2464	*/
2465	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2466	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2467	("fCtxChanged=%#RX64\n", fCtxChanged));
2468
2469	/*
2470	* If we need to log state that isn't always imported, we'll need to import them here.
2471	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2472	*/
2473	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2474	#endif
2475
2476	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2477	return VINF_SUCCESS;
2478	}
2479
2480	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2481
2482	/**
2483	* Merges the guest and nested-guest MSR permission bitmap.
2484	*
2485	* If the guest is intercepting an MSR we need to intercept it regardless of
2486	* whether the nested-guest is intercepting it or not.
2487	*
2488	* @param pHostCpu The HM physical-CPU structure.
2489	* @param pVCpu The cross context virtual CPU structure.
2490	*
2491	* @remarks No-long-jmp zone!!!
2492	*/
2493	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2494	{
2495	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2496	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2497	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2498
2499	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2500	uint32_t const offRsvdQwords = 0x1800 >> 3;
2501	for (uint32_t i = 0; i < offRsvdQwords; i++)
2502	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2503	}
2504
2505
2506	/**
2507	* Caches the nested-guest VMCB fields before we modify them for execution using
2508	* hardware-assisted SVM.
2509	*
2510	* @returns true if the VMCB was previously already cached, false otherwise.
2511	* @param pVCpu The cross context virtual CPU structure.
2512	*
2513	* @sa HMNotifySvmNstGstVmexit.
2514	*/
2515	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2516	{
2517	/*
2518	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2519	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2520	*
2521	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2522	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2523	*/
2524	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2525	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2526	if (!fWasCached)
2527	{
2528	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2529	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2530	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2531	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2532	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2533	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2534	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2535	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2536	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2537	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2538	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2539	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2540	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2541	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2542	pVmcbNstGstCache->fCacheValid = true;
2543	Log4Func(("Cached VMCB fields\n"));
2544	}
2545
2546	return fWasCached;
2547	}
2548
2549
2550	/**
2551	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2552	*
2553	* This is done the first time we enter nested-guest execution using SVM R0
2554	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2555	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2556	*
2557	* @param pVCpu The cross context virtual CPU structure.
2558	*/
2559	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2560	{
2561	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2562	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2563
2564	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2565
2566	/*
2567	* First cache the nested-guest VMCB fields we may potentially modify.
2568	*/
2569	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2570	if (!fVmcbCached)
2571	{
2572	/*
2573	* The IOPM of the nested-guest can be ignored because the the guest always
2574	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2575	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2576	*/
2577	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2578
2579	/*
2580	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2581	* nested-paging suddenly while executing a VM (see assertion at the end of
2582	* Trap0eHandler() in PGMAllBth.h).
2583	*/
2584	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2585
2586	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2587	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2588
2589	/*
2590	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2591	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2592	*/
2593	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2594
2595	# ifdef DEBUG_ramshankar
2596	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2597	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2598	# endif
2599
2600	/*
2601	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2602	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2603	*/
2604	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2605	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2606	\| SVM_CTRL_INTERCEPT_VMLOAD;
2607
2608	/*
2609	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2610	* CLGI/STGI instructions executed by the nested-guest.
2611	*/
2612	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2613	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2614	\| SVM_CTRL_INTERCEPT_STGI;
2615
2616	/* Merge the guest and nested-guest intercepts. */
2617	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2618
2619	/* Update the VMCB clean bits. */
2620	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2621	}
2622	else
2623	{
2624	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2625	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2626	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2627	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2628	}
2629	}
2630
2631	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2632
2633	/**
2634	* Exports the state shared between the host and guest (or nested-guest) into
2635	* the VMCB.
2636	*
2637	* @param pVCpu The cross context virtual CPU structure.
2638	* @param pVmcb Pointer to the VM control block.
2639	*
2640	* @remarks No-long-jump zone!!!
2641	*/
2642	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2643	{
2644	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2645	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2646
2647	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2648	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2649
2650	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2651	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2652	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2653	}
2654
2655
2656	/**
2657	* Worker for SVMR0ImportStateOnDemand.
2658	*
2659	* @param pVCpu The cross context virtual CPU structure.
2660	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2661	*/
2662	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2663	{
2664	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2665
2666	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2667	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2668	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2669	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2670
2671	/*
2672	* We disable interrupts to make the updating of the state and in particular
2673	* the fExtrn modification atomic wrt to preemption hooks.
2674	*/
2675	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2676
2677	fWhat &= pCtx->fExtrn;
2678	if (fWhat)
2679	{
2680	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2681	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2682	{
2683	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2684	{
2685	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2686	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2687	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2688	}
2689	}
2690
2691	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2692	{
2693	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2694	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2695	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2696	}
2697	#endif
2698
2699	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2700	CPUMUpdateInterruptShadowEx(pCtx, pVmcbCtrl->IntShadow.n.u1IntShadow, pVmcbGuest->u64RIP);
2701
2702	if (fWhat & CPUMCTX_EXTRN_RIP)
2703	pCtx->rip = pVmcbGuest->u64RIP;
2704
2705	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2706	{
2707	pCtx->eflags.u = pVmcbGuest->u64RFlags;
2708	if (pVCpu->hmr0.s.fClearTrapFlag)
2709	{
2710	pVCpu->hmr0.s.fClearTrapFlag = false;
2711	pCtx->eflags.Bits.u1TF = 0;
2712	}
2713	}
2714
2715	if (fWhat & CPUMCTX_EXTRN_RSP)
2716	pCtx->rsp = pVmcbGuest->u64RSP;
2717
2718	if (fWhat & CPUMCTX_EXTRN_RAX)
2719	pCtx->rax = pVmcbGuest->u64RAX;
2720
2721	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2722	{
2723	if (fWhat & CPUMCTX_EXTRN_CS)
2724	{
2725	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2726	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2727	/** @todo SELM might need to be fixed as it too should not care about the
2728	* granularity bit. See @bugref{6785}. */
2729	if ( !pCtx->cs.Attr.n.u1Granularity
2730	&& pCtx->cs.Attr.n.u1Present
2731	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2732	{
2733	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2734	pCtx->cs.Attr.n.u1Granularity = 1;
2735	}
2736	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2737	}
2738	if (fWhat & CPUMCTX_EXTRN_SS)
2739	{
2740	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2741	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2742	/*
2743	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2744	* VMCB and uses that and thus it's possible that when the CPL changes during
2745	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2746	* AMD Fusion CPUs with 64-bit guests.
2747	*
2748	* See AMD spec. 15.5.1 "Basic operation".
2749	*/
2750	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2751	uint8_t const uCpl = pVmcbGuest->u8CPL;
2752	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2753	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2754	}
2755	if (fWhat & CPUMCTX_EXTRN_DS)
2756	{
2757	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2758	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2759	}
2760	if (fWhat & CPUMCTX_EXTRN_ES)
2761	{
2762	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2763	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2764	}
2765	if (fWhat & CPUMCTX_EXTRN_FS)
2766	{
2767	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2768	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2769	}
2770	if (fWhat & CPUMCTX_EXTRN_GS)
2771	{
2772	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2773	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2774	}
2775	}
2776
2777	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2778	{
2779	if (fWhat & CPUMCTX_EXTRN_TR)
2780	{
2781	/*
2782	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2783	* are used between Intel and AMD, see @bugref{6208#c39}.
2784	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2785	*/
2786	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2787	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2788	{
2789	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2790	\|\| CPUMIsGuestInLongModeEx(pCtx))
2791	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2792	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2793	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2794	}
2795	}
2796
2797	if (fWhat & CPUMCTX_EXTRN_LDTR)
2798	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2799
2800	if (fWhat & CPUMCTX_EXTRN_GDTR)
2801	{
2802	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2803	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2804	}
2805
2806	if (fWhat & CPUMCTX_EXTRN_IDTR)
2807	{
2808	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2809	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2810	}
2811	}
2812
2813	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2814	{
2815	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2816	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2817	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2818	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2819	}
2820
2821	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2822	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2823	{
2824	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2825	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2826	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2827	}
2828
2829	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2830	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2831
2832	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2833	{
2834	if (fWhat & CPUMCTX_EXTRN_DR6)
2835	{
2836	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2837	pCtx->dr[6] = pVmcbGuest->u64DR6;
2838	else
2839	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2840	}
2841
2842	if (fWhat & CPUMCTX_EXTRN_DR7)
2843	{
2844	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2845	pCtx->dr[7] = pVmcbGuest->u64DR7;
2846	else
2847	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2848	}
2849	}
2850
2851	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2852	{
2853	if (fWhat & CPUMCTX_EXTRN_CR0)
2854	{
2855	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2856	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2857	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2858	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2859	CPUMSetGuestCR0(pVCpu, uCr0);
2860	VMMRZCallRing3Enable(pVCpu);
2861	}
2862
2863	if (fWhat & CPUMCTX_EXTRN_CR2)
2864	pCtx->cr2 = pVmcbGuest->u64CR2;
2865
2866	if (fWhat & CPUMCTX_EXTRN_CR3)
2867	{
2868	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2869	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2870	{
2871	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2872	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2873	}
2874	}
2875
2876	/* Changes to CR4 are always intercepted. */
2877	}
2878
2879	/* Update fExtrn. */
2880	pCtx->fExtrn &= ~fWhat;
2881
2882	/* If everything has been imported, clear the HM keeper bit. */
2883	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2884	{
2885	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2886	Assert(!pCtx->fExtrn);
2887	}
2888	}
2889	else
2890	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2891
2892	ASMSetFlags(fEFlags);
2893
2894	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2895
2896	/*
2897	* Honor any pending CR3 updates.
2898	*
2899	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2900	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2901	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2902	* handling -> hmR0SvmImportGuestState() and here we are.
2903	*
2904	* The reason for such complicated handling is because VM-exits that call into PGM expect
2905	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2906	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2907	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2908	*
2909	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2910	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2911	*/
2912	if ( VMMRZCallRing3IsEnabled(pVCpu)
2913	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2914	{
2915	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2916	PGMUpdateCR3(pVCpu, pCtx->cr3);
2917	}
2918	}
2919
2920
2921	/**
2922	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2923	* context.
2924	*
2925	* Currently there is no residual state left in the CPU that is not updated in the
2926	* VMCB.
2927	*
2928	* @returns VBox status code.
2929	* @param pVCpu The cross context virtual CPU structure.
2930	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2931	*/
2932	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2933	{
2934	hmR0SvmImportGuestState(pVCpu, fWhat);
2935	return VINF_SUCCESS;
2936	}
2937
2938
2939	/**
2940	* Gets SVM \#VMEXIT auxiliary information.
2941	*
2942	* @returns VBox status code.
2943	* @param pVCpu The cross context virtual CPU structure.
2944	* @param pSvmExitAux Where to store the auxiliary info.
2945	*/
2946	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2947	{
2948	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2949	if (RT_LIKELY(pSvmTransient))
2950	{
2951	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2952	if (RT_LIKELY(pVmcb))
2953	{
2954	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2955	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2956	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2957	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2958	return VINF_SUCCESS;
2959	}
2960	return VERR_SVM_IPE_5;
2961	}
2962	return VERR_NOT_AVAILABLE;
2963	}
2964
2965
2966	/**
2967	* Does the necessary state syncing before returning to ring-3 for any reason
2968	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2969	*
2970	* @param pVCpu The cross context virtual CPU structure.
2971	* @param fImportState Whether to import the guest state from the VMCB back
2972	* to the guest-CPU context.
2973	*
2974	* @remarks No-long-jmp zone!!!
2975	*/
2976	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2977	{
2978	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2979	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2980
2981	/*
2982	* !!! IMPORTANT !!!
2983	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2984	*/
2985
2986	/* Save the guest state if necessary. */
2987	if (fImportState)
2988	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2989
2990	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2991	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2992	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2993
2994	/*
2995	* Restore host debug registers if necessary and resync on next R0 reentry.
2996	*/
2997	#ifdef VBOX_STRICT
2998	if (CPUMIsHyperDebugStateActive(pVCpu))
2999	{
3000	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
3001	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
3002	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
3003	}
3004	#endif
3005	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3006	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3007	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3008
3009	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3010	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3011	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3012	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3013	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3014	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3015	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3016
3017	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3018	}
3019
3020
3021	/**
3022	* Leaves the AMD-V session.
3023	*
3024	* Only used while returning to ring-3 either due to longjump or exits to
3025	* ring-3.
3026	*
3027	* @returns VBox status code.
3028	* @param pVCpu The cross context virtual CPU structure.
3029	*/
3030	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3031	{
3032	HM_DISABLE_PREEMPT(pVCpu);
3033	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3034	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3035
3036	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3037	and done this from the SVMR0ThreadCtxCallback(). */
3038	if (!pVCpu->hmr0.s.fLeaveDone)
3039	{
3040	hmR0SvmLeave(pVCpu, true /* fImportState */);
3041	pVCpu->hmr0.s.fLeaveDone = true;
3042	}
3043
3044	/*
3045	* !!! IMPORTANT !!!
3046	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3047	*/
3048
3049	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3050	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3051	VMMR0ThreadCtxHookDisable(pVCpu);
3052
3053	/* Leave HM context. This takes care of local init (term). */
3054	int rc = HMR0LeaveCpu(pVCpu);
3055
3056	HM_RESTORE_PREEMPT();
3057	return rc;
3058	}
3059
3060
3061	/**
3062	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3063	* any remaining host state) before we go back to ring-3 due to an assertion.
3064	*
3065	* @param pVCpu The cross context virtual CPU structure.
3066	*/
3067	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3068	{
3069	/*
3070	* !!! IMPORTANT !!!
3071	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3072	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3073	*/
3074	VMMR0AssertionRemoveNotification(pVCpu);
3075	VMMRZCallRing3Disable(pVCpu);
3076	HM_DISABLE_PREEMPT(pVCpu);
3077
3078	/* Import the entire guest state. */
3079	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3080
3081	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3082	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3083
3084	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3085	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3086
3087	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3088	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3089	VMMR0ThreadCtxHookDisable(pVCpu);
3090
3091	/* Leave HM context. This takes care of local init (term). */
3092	HMR0LeaveCpu(pVCpu);
3093
3094	HM_RESTORE_PREEMPT();
3095	return VINF_SUCCESS;
3096	}
3097
3098
3099	/**
3100	* Take necessary actions before going back to ring-3.
3101	*
3102	* An action requires us to go back to ring-3. This function does the necessary
3103	* steps before we can safely return to ring-3. This is not the same as longjmps
3104	* to ring-3, this is voluntary.
3105	*
3106	* @returns Strict VBox status code.
3107	* @param pVCpu The cross context virtual CPU structure.
3108	* @param rcExit The reason for exiting to ring-3. Can be
3109	* VINF_VMM_UNKNOWN_RING3_CALL.
3110	*/
3111	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3112	{
3113	Assert(pVCpu);
3114	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3115
3116	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3117	VMMRZCallRing3Disable(pVCpu);
3118	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3119	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3120
3121	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3122	if (pVCpu->hm.s.Event.fPending)
3123	{
3124	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3125	Assert(!pVCpu->hm.s.Event.fPending);
3126	}
3127
3128	/* Sync. the necessary state for going back to ring-3. */
3129	hmR0SvmLeaveSession(pVCpu);
3130	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3131
3132	/* Thread-context hooks are unregistered at this point!!! */
3133	/* Ring-3 callback notifications are unregistered at this point!!! */
3134
3135	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3136	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3137	\| CPUM_CHANGED_LDTR
3138	\| CPUM_CHANGED_GDTR
3139	\| CPUM_CHANGED_IDTR
3140	\| CPUM_CHANGED_TR
3141	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3142	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3143	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3144	{
3145	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3146	}
3147
3148	/* Update the exit-to-ring 3 reason. */
3149	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3150
3151	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3152	if ( rcExit != VINF_EM_RAW_INTERRUPT
3153	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3154	{
3155	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3156	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3157	}
3158
3159	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3160	VMMRZCallRing3Enable(pVCpu);
3161
3162	/*
3163	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3164	* and if we're injecting an event we should have a TRPM trap pending.
3165	*/
3166	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3167	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3168	VERR_SVM_IPE_5);
3169	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3170	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3171	VERR_SVM_IPE_4);
3172
3173	return rcExit;
3174	}
3175
3176
3177	/**
3178	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3179	* intercepts.
3180	*
3181	* @param pVCpu The cross context virtual CPU structure.
3182	* @param pVmcb Pointer to the VM control block.
3183	*
3184	* @remarks No-long-jump zone!!!
3185	*/
3186	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3187	{
3188	/*
3189	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3190	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3191	* RDTSC/RDTSCP as well.
3192	*/
3193	bool fParavirtTsc;
3194	uint64_t uTscOffset;
3195	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3196
3197	bool fIntercept;
3198	if (fCanUseRealTsc)
3199	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3200	else
3201	{
3202	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3203	fIntercept = true;
3204	}
3205
3206	if (!fIntercept)
3207	{
3208	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3209	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3210	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3211	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3212	#endif
3213
3214	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3215	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3216	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3217	}
3218
3219	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3220	information before every VM-entry, hence we have nothing to do here at the moment. */
3221	if (fParavirtTsc)
3222	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3223	}
3224
3225
3226	/**
3227	* Sets an event as a pending event to be injected into the guest.
3228	*
3229	* @param pVCpu The cross context virtual CPU structure.
3230	* @param pEvent Pointer to the SVM event.
3231	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3232	* page-fault.
3233	*
3234	* @remarks Statistics counter assumes this is a guest event being reflected to
3235	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3236	*/
3237	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3238	{
3239	Assert(!pVCpu->hm.s.Event.fPending);
3240	Assert(pEvent->n.u1Valid);
3241
3242	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3243	pVCpu->hm.s.Event.fPending = true;
3244	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3245
3246	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3247	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3248	}
3249
3250
3251	/**
3252	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3253	*
3254	* @param pVCpu The cross context virtual CPU structure.
3255	*/
3256	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3257	{
3258	SVMEVENT Event;
3259	Event.u = 0;
3260	Event.n.u1Valid = 1;
3261	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3262	Event.n.u8Vector = X86_XCPT_DE;
3263	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3264	}
3265
3266
3267	/**
3268	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3269	*
3270	* @param pVCpu The cross context virtual CPU structure.
3271	*/
3272	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3273	{
3274	SVMEVENT Event;
3275	Event.u = 0;
3276	Event.n.u1Valid = 1;
3277	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3278	Event.n.u8Vector = X86_XCPT_UD;
3279	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3280	}
3281
3282
3283	/**
3284	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3285	*
3286	* @param pVCpu The cross context virtual CPU structure.
3287	*/
3288	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3289	{
3290	SVMEVENT Event;
3291	Event.u = 0;
3292	Event.n.u1Valid = 1;
3293	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3294	Event.n.u8Vector = X86_XCPT_DB;
3295	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3296	}
3297
3298
3299	/**
3300	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3301	*
3302	* @param pVCpu The cross context virtual CPU structure.
3303	* @param u32ErrCode The error-code for the page-fault.
3304	* @param uFaultAddress The page fault address (CR2).
3305	*
3306	* @remarks This updates the guest CR2 with @a uFaultAddress!
3307	*/
3308	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3309	{
3310	SVMEVENT Event;
3311	Event.u = 0;
3312	Event.n.u1Valid = 1;
3313	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3314	Event.n.u8Vector = X86_XCPT_PF;
3315	Event.n.u1ErrorCodeValid = 1;
3316	Event.n.u32ErrorCode = u32ErrCode;
3317
3318	/* Update CR2 of the guest. */
3319	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3320	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3321	{
3322	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3323	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3324	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3325	}
3326
3327	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3328	}
3329
3330
3331	/**
3332	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3333	*
3334	* @param pVCpu The cross context virtual CPU structure.
3335	*/
3336	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3337	{
3338	SVMEVENT Event;
3339	Event.u = 0;
3340	Event.n.u1Valid = 1;
3341	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3342	Event.n.u8Vector = X86_XCPT_MF;
3343	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3344	}
3345
3346
3347	/**
3348	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3349	*
3350	* @param pVCpu The cross context virtual CPU structure.
3351	*/
3352	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3353	{
3354	SVMEVENT Event;
3355	Event.u = 0;
3356	Event.n.u1Valid = 1;
3357	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3358	Event.n.u8Vector = X86_XCPT_DF;
3359	Event.n.u1ErrorCodeValid = 1;
3360	Event.n.u32ErrorCode = 0;
3361	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3362	}
3363
3364
3365	/**
3366	* Injects an event into the guest upon VMRUN by updating the relevant field
3367	* in the VMCB.
3368	*
3369	* @param pVCpu The cross context virtual CPU structure.
3370	* @param pVmcb Pointer to the guest VM control block.
3371	* @param pEvent Pointer to the event.
3372	*
3373	* @remarks No-long-jump zone!!!
3374	* @remarks Requires CR0!
3375	*/
3376	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3377	{
3378	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3379	pVmcb->ctrl.EventInject.u = pEvent->u;
3380	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3381	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3382	{
3383	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3384	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3385	}
3386	else
3387	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3388	RT_NOREF(pVCpu);
3389
3390	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3391	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3392	}
3393
3394
3395
3396	/**
3397	* Converts any TRPM trap into a pending HM event. This is typically used when
3398	* entering from ring-3 (not longjmp returns).
3399	*
3400	* @param pVCpu The cross context virtual CPU structure.
3401	*/
3402	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3403	{
3404	Assert(TRPMHasTrap(pVCpu));
3405	Assert(!pVCpu->hm.s.Event.fPending);
3406
3407	uint8_t uVector;
3408	TRPMEVENT enmTrpmEvent;
3409	uint32_t uErrCode;
3410	RTGCUINTPTR GCPtrFaultAddress;
3411	uint8_t cbInstr;
3412
3413	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3414	AssertRC(rc);
3415
3416	SVMEVENT Event;
3417	Event.u = 0;
3418	Event.n.u1Valid = 1;
3419	Event.n.u8Vector = uVector;
3420
3421	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3422	if (enmTrpmEvent == TRPM_TRAP)
3423	{
3424	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3425	switch (uVector)
3426	{
3427	case X86_XCPT_BP:
3428	case X86_XCPT_OF:
3429	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3430	RT_FALL_THRU();
3431
3432	case X86_XCPT_PF:
3433	case X86_XCPT_DF:
3434	case X86_XCPT_TS:
3435	case X86_XCPT_NP:
3436	case X86_XCPT_SS:
3437	case X86_XCPT_GP:
3438	case X86_XCPT_AC:
3439	{
3440	Event.n.u1ErrorCodeValid = 1;
3441	Event.n.u32ErrorCode = uErrCode;
3442	break;
3443	}
3444	}
3445	}
3446	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3447	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3448	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3449	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3450	else if (enmTrpmEvent == TRPM_NMI)
3451	Event.n.u3Type = SVM_EVENT_NMI;
3452	else
3453	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3454
3455	rc = TRPMResetTrap(pVCpu);
3456	AssertRC(rc);
3457
3458	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3459	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3460
3461	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3462	}
3463
3464
3465	/**
3466	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3467	* AMD-V to execute any instruction.
3468	*
3469	* @param pVCpu The cross context virtual CPU structure.
3470	*/
3471	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3472	{
3473	Assert(pVCpu->hm.s.Event.fPending);
3474	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3475
3476	SVMEVENT Event;
3477	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3478
3479	uint8_t uVector = Event.n.u8Vector;
3480	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3481
3482	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3483
3484	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3485	AssertRC(rc);
3486
3487	if (Event.n.u1ErrorCodeValid)
3488	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3489
3490	if ( enmTrapType == TRPM_TRAP
3491	&& uVector == X86_XCPT_PF)
3492	{
3493	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3494	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3495	}
3496	else if (enmTrapType == TRPM_SOFTWARE_INT)
3497	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3498	pVCpu->hm.s.Event.fPending = false;
3499	}
3500
3501
3502	/**
3503	* Sets the virtual interrupt intercept control in the VMCB.
3504	*
3505	* @param pVCpu The cross context virtual CPU structure.
3506	* @param pVmcb Pointer to the VM control block.
3507	*/
3508	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3509	{
3510	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3511
3512	/*
3513	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3514	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3515	* (updating ISR at the right time) and inject the interrupt.
3516	*
3517	* With AVIC is supported, we could make use of the asynchronously delivery without
3518	* #VMEXIT and we would be passing the AVIC page to SVM.
3519	*
3520	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3521	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3522	*/
3523	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3524	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3525	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3526	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3527	Log4(("Set VINTR intercept\n"));
3528	}
3529
3530
3531	/**
3532	* Clears the virtual interrupt intercept control in the VMCB as
3533	* we are figured the guest is unable process any interrupts
3534	* at this point of time.
3535	*
3536	* @param pVCpu The cross context virtual CPU structure.
3537	* @param pVmcb Pointer to the VM control block.
3538	*/
3539	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3540	{
3541	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3542
3543	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3544	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3545	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3546	{
3547	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3548	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3549	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3550	Log4(("Cleared VINTR intercept\n"));
3551	}
3552	}
3553
3554
3555	/**
3556	* Evaluates the event to be delivered to the guest and sets it as the pending
3557	* event.
3558	*
3559	* @returns Strict VBox status code.
3560	* @param pVCpu The cross context virtual CPU structure.
3561	* @param pSvmTransient Pointer to the SVM transient structure.
3562	*/
3563	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3564	{
3565	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3566	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3567	\| CPUMCTX_EXTRN_RFLAGS
3568	\| CPUMCTX_EXTRN_INHIBIT_INT
3569	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3570
3571	Assert(!pVCpu->hm.s.Event.fPending);
3572	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3573	Assert(pVmcb);
3574
3575	bool const fGif = CPUMGetGuestGif(pCtx);
3576	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(pCtx);
3577	bool const fBlockNmi = CPUMAreInterruptsInhibitedByNmi(pCtx);
3578
3579	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3580	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3581	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3582
3583	/** @todo SMI. SMIs take priority over NMIs. */
3584
3585	/*
3586	* Check if the guest or nested-guest can receive NMIs.
3587	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3588	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3589	*/
3590	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3591	&& !fBlockNmi)
3592	{
3593	if ( fGif
3594	&& !fIntShadow)
3595	{
3596	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3597	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3598	{
3599	Log4(("Intercepting NMI -> #VMEXIT\n"));
3600	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3601	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3602	}
3603	#endif
3604	Log4(("Setting NMI pending for injection\n"));
3605	SVMEVENT Event;
3606	Event.u = 0;
3607	Event.n.u1Valid = 1;
3608	Event.n.u8Vector = X86_XCPT_NMI;
3609	Event.n.u3Type = SVM_EVENT_NMI;
3610	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3611	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3612	}
3613	else if (!fGif)
3614	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3615	else if (!pSvmTransient->fIsNestedGuest)
3616	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3617	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3618	}
3619	/*
3620	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3621	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3622	* it from the APIC device.
3623	*
3624	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3625	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3626	* do that work when we execute nested-guest code esp. since all the required information
3627	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3628	* the virtual interrupt controller.
3629	*
3630	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3631	*/
3632	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3633	&& !pVCpu->hm.s.fSingleInstruction)
3634	{
3635	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u & X86_EFL_IF)
3636	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3637	if ( fGif
3638	&& !fBlockInt
3639	&& !fIntShadow)
3640	{
3641	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3642	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3643	{
3644	Log4(("Intercepting INTR -> #VMEXIT\n"));
3645	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3646	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3647	}
3648	#endif
3649	uint8_t u8Interrupt;
3650	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3651	if (RT_SUCCESS(rc))
3652	{
3653	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3654	SVMEVENT Event;
3655	Event.u = 0;
3656	Event.n.u1Valid = 1;
3657	Event.n.u8Vector = u8Interrupt;
3658	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3659	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3660	}
3661	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3662	{
3663	/*
3664	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3665	* updated eventually when the TPR is written by the guest.
3666	*/
3667	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3668	}
3669	else
3670	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3671	}
3672	else if (!fGif)
3673	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3674	else if (!pSvmTransient->fIsNestedGuest)
3675	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3676	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3677	}
3678
3679	return VINF_SUCCESS;
3680	}
3681
3682
3683	/**
3684	* Injects any pending events into the guest (or nested-guest).
3685	*
3686	* @param pVCpu The cross context virtual CPU structure.
3687	* @param pVmcb Pointer to the VM control block.
3688	*
3689	* @remarks Must only be called when we are guaranteed to enter
3690	* hardware-assisted SVM execution and not return to ring-3
3691	* prematurely.
3692	*/
3693	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3694	{
3695	Assert(!TRPMHasTrap(pVCpu));
3696	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3697
3698	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx);
3699	#ifdef VBOX_STRICT
3700	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3701	bool const fGif = CPUMGetGuestGif(pCtx);
3702	bool fAllowInt = fGif;
3703	if (fGif)
3704	{
3705	/*
3706	* For nested-guests we have no way to determine if we're injecting a physical or
3707	* virtual interrupt at this point. Hence the partial verification below.
3708	*/
3709	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3710	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3711	else
3712	fAllowInt = RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
3713	}
3714	#endif
3715
3716	if (pVCpu->hm.s.Event.fPending)
3717	{
3718	SVMEVENT Event;
3719	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3720	Assert(Event.n.u1Valid);
3721
3722	/*
3723	* Validate event injection pre-conditions.
3724	*/
3725	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3726	{
3727	Assert(fAllowInt);
3728	Assert(!fIntShadow);
3729	}
3730	else if (Event.n.u3Type == SVM_EVENT_NMI)
3731	{
3732	Assert(fGif);
3733	Assert(!fIntShadow);
3734	}
3735
3736	/*
3737	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3738	* do this only when we are surely going to inject the NMI as otherwise if we return
3739	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3740	* SVM R0.
3741	*
3742	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3743	* which will set the VMCS field after actually delivering the NMI which we read on
3744	* VM-exit to determine the state.
3745	*/
3746	if ( Event.n.u3Type == SVM_EVENT_NMI
3747	&& Event.n.u8Vector == X86_XCPT_NMI)
3748	CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3749
3750	/*
3751	* Inject it (update VMCB for injection by the hardware).
3752	*/
3753	Log4(("Injecting pending HM event\n"));
3754	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3755	pVCpu->hm.s.Event.fPending = false;
3756
3757	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3758	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3759	else
3760	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3761	}
3762	else
3763	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3764
3765	/*
3766	* We could have injected an NMI through IEM and continue guest execution using
3767	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3768	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3769	*/
3770	if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
3771	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3772
3773	/*
3774	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3775	*
3776	* For nested-guests: We need to update it too for the scenario where IEM executes
3777	* the nested-guest but execution later continues here with an interrupt shadow active.
3778	*/
3779	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3780	}
3781
3782
3783	/**
3784	* Reports world-switch error and dumps some useful debug info.
3785	*
3786	* @param pVCpu The cross context virtual CPU structure.
3787	* @param rcVMRun The return code from VMRUN (or
3788	* VERR_SVM_INVALID_GUEST_STATE for invalid
3789	* guest-state).
3790	*/
3791	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3792	{
3793	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3794	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3795	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3796
3797	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3798	{
3799	#ifdef VBOX_STRICT
3800	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3801	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3802	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3803	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3804	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3805	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3806	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3807	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3808	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3809	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3810	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3811	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3812
3813	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3814	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3815	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3816
3817	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3818	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3819	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3820	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3821	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3822	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3823	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3824	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3825	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3826	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3827	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3828	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3829
3830	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3831	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3832	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3833	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3834	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3835	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3836	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3837	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3838	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3839	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3840	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3841	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3842	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3843	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3844	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3845	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3846	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3847	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3848	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3849	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3850
3851	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3852
3853	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3854	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3855
3856	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3857	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3858	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3859	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3860	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3861	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3862	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3863	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3864	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3865	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3866	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3867	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3868	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3869	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3870	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3871	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3872	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3873	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3874	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3875	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3876
3877	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3878	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3879
3880	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3881	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3882	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3883	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3884
3885	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3886	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3887
3888	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3889	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3890	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3891	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3892
3893	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3894	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3895	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3896	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3897	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3898	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3899	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3900
3901	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3902	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3903	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3904	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3905
3906	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3907	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3908	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3909
3910	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3911	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3912	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3913	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3914	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3915	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3916	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3917	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3918	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3919	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3920	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3921	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3922
3923	NOREF(pVmcb);
3924	#endif /* VBOX_STRICT */
3925	}
3926	else
3927	Log4Func(("rcVMRun=%d\n", rcVMRun));
3928	}
3929
3930
3931	/**
3932	* Check per-VM and per-VCPU force flag actions that require us to go back to
3933	* ring-3 for one reason or another.
3934	*
3935	* @returns Strict VBox status code (information status code included).
3936	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3937	* ring-3.
3938	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3939	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3940	* interrupts)
3941	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3942	* all EMTs to be in ring-3.
3943	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3944	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3945	* to the EM loop.
3946	*
3947	* @param pVCpu The cross context virtual CPU structure.
3948	*/
3949	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3950	{
3951	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3952
3953	/* Could happen as a result of longjump. */
3954	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3955	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3956
3957	/* Update pending interrupts into the APIC's IRR. */
3958	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3959	PDMApicUpdatePendingInterrupts(pVCpu);
3960
3961	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3962	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3963	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3964	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3965	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3966	{
3967	/* Pending PGM C3 sync. */
3968	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3969	{
3970	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3971	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3972	if (rc != VINF_SUCCESS)
3973	{
3974	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3975	return rc;
3976	}
3977	}
3978
3979	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3980	/* -XXX- what was that about single stepping? */
3981	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3982	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3983	{
3984	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3985	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3986	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3987	return rc;
3988	}
3989
3990	/* Pending VM request packets, such as hardware interrupts. */
3991	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3992	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3993	{
3994	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
3995	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3996	return VINF_EM_PENDING_REQUEST;
3997	}
3998
3999	/* Pending PGM pool flushes. */
4000	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4001	{
4002	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4003	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4004	return VINF_PGM_POOL_FLUSH_PENDING;
4005	}
4006
4007	/* Pending DMA requests. */
4008	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4009	{
4010	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4011	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4012	return VINF_EM_RAW_TO_R3;
4013	}
4014	}
4015
4016	return VINF_SUCCESS;
4017	}
4018
4019
4020	/**
4021	* Does the preparations before executing guest code in AMD-V.
4022	*
4023	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4024	* recompiler. We must be cautious what we do here regarding committing
4025	* guest-state information into the VMCB assuming we assuredly execute the guest
4026	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4027	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4028	* that the recompiler can (and should) use them when it resumes guest
4029	* execution. Otherwise such operations must be done when we can no longer
4030	* exit to ring-3.
4031	*
4032	* @returns Strict VBox status code (informational status codes included).
4033	* @retval VINF_SUCCESS if we can proceed with running the guest.
4034	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4035	*
4036	* @param pVCpu The cross context virtual CPU structure.
4037	* @param pSvmTransient Pointer to the SVM transient structure.
4038	*/
4039	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4040	{
4041	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4042
4043	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4044	if (pSvmTransient->fIsNestedGuest)
4045	{
4046	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4047	return VINF_EM_RESCHEDULE_REM;
4048	}
4049	#endif
4050
4051	/* Check force flag actions that might require us to go back to ring-3. */
4052	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4053	if (rc != VINF_SUCCESS)
4054	return rc;
4055
4056	if (TRPMHasTrap(pVCpu))
4057	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4058	else if (!pVCpu->hm.s.Event.fPending)
4059	{
4060	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4061	if ( rc != VINF_SUCCESS
4062	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4063	{
4064	/* If a nested-guest VM-exit occurred, bail. */
4065	if (pSvmTransient->fIsNestedGuest)
4066	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4067	return rc;
4068	}
4069	}
4070
4071	/*
4072	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4073	* Just do it in software, see @bugref{8411}.
4074	* NB: If we could continue a task switch exit we wouldn't need to do this.
4075	*/
4076	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4077	if (RT_UNLIKELY( !g_fHmSvmFeatures
4078	&& pVCpu->hm.s.Event.fPending
4079	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4080	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4081
4082	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4083	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4084	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4085	#endif
4086
4087	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4088	/*
4089	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4090	*/
4091	if (pSvmTransient->fIsNestedGuest)
4092	hmR0SvmSetupVmcbNested(pVCpu);
4093	#endif
4094
4095	/*
4096	* Export the guest state bits that are not shared with the host in any way as we can
4097	* longjmp or get preempted in the midst of exporting some of the state.
4098	*/
4099	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4100	AssertRCReturn(rc, rc);
4101	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4102
4103	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4104	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4105
4106	/*
4107	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4108	* world-switch so we can update it on the way back if the guest changed the TPR.
4109	*/
4110	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4111	{
4112	Assert(!pSvmTransient->fIsNestedGuest);
4113	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4114	if (pVM->hm.s.fTprPatchingActive)
4115	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4116	else
4117	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4118	}
4119
4120	/*
4121	* No longjmps to ring-3 from this point on!!!
4122	*
4123	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4124	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4125	*/
4126	VMMRZCallRing3Disable(pVCpu);
4127
4128	/*
4129	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4130	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4131	* preemption disabled for a while. Since this is purly to aid the
4132	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4133	* disable interrupt on NT.
4134	*
4135	* We need to check for force-flags that could've possible been altered since we last
4136	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4137	* see @bugref{6398}).
4138	*
4139	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4140	* to ring-3 before executing guest code.
4141	*/
4142	pSvmTransient->fEFlags = ASMIntDisableFlags();
4143	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4144	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4145	{
4146	ASMSetFlags(pSvmTransient->fEFlags);
4147	VMMRZCallRing3Enable(pVCpu);
4148	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4149	return VINF_EM_RAW_TO_R3;
4150	}
4151	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4152	{
4153	ASMSetFlags(pSvmTransient->fEFlags);
4154	VMMRZCallRing3Enable(pVCpu);
4155	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4156	return VINF_EM_RAW_INTERRUPT;
4157	}
4158
4159	return VINF_SUCCESS;
4160	}
4161
4162
4163	/**
4164	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4165	* doing so.
4166	*
4167	* This means there is no backing out to ring-3 or anywhere else at this point.
4168	*
4169	* @param pVCpu The cross context virtual CPU structure.
4170	* @param pSvmTransient Pointer to the SVM transient structure.
4171	*
4172	* @remarks Called with preemption disabled.
4173	* @remarks No-long-jump zone!!!
4174	*/
4175	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4176	{
4177	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4179
4180	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4181	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4182
4183	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4184	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4185
4186	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4187
4188	if (!CPUMIsGuestFPUStateActive(pVCpu))
4189	{
4190	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4191	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4192	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4193	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4194	}
4195
4196	/* Load the state shared between host and guest (FPU, debug). */
4197	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4198	hmR0SvmExportSharedState(pVCpu, pVmcb);
4199
4200	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4201	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4202
4203	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4204	RTCPUID const idHostCpu = pHostCpu->idCpu;
4205	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4206
4207	/* Setup TSC offsetting. */
4208	if ( pSvmTransient->fUpdateTscOffsetting
4209	\|\| fMigratedHostCpu)
4210	{
4211	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4212	pSvmTransient->fUpdateTscOffsetting = false;
4213	}
4214
4215	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4216	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4217	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4218	else
4219	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4220
4221	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4222	if (fMigratedHostCpu)
4223	pVmcb->ctrl.u32VmcbCleanBits = 0;
4224
4225	/* Store status of the shared guest-host state at the time of VMRUN. */
4226	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4227	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4228
4229	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4230	uint8_t *pbMsrBitmap;
4231	if (!pSvmTransient->fIsNestedGuest)
4232	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4233	else
4234	{
4235	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4236	* MSRPM and only perform this if it changed also use EVEX.POR when it
4237	* does. */
4238	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4239
4240	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4241	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4242	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4243	}
4244	#else
4245	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4246	#endif
4247
4248	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4249	/* Flush the appropriate tagged-TLB entries. */
4250	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4251	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4252
4253	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4254
4255	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4256	to start executing. */
4257
4258	/*
4259	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4260	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4261	*
4262	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4263	*/
4264	if ( g_CpumHostFeatures.s.fRdTscP
4265	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4266	{
4267	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4268	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4269	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4270	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4271	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4272	pSvmTransient->fRestoreTscAuxMsr = true;
4273	}
4274	else
4275	{
4276	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4277	pSvmTransient->fRestoreTscAuxMsr = false;
4278	}
4279	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4280
4281	/*
4282	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4283	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4284	* from the VMCB.
4285	*/
4286	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4287	if (!fSupportsVmcbCleanBits)
4288	pVmcb->ctrl.u32VmcbCleanBits = 0;
4289	}
4290
4291
4292	/**
4293	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4294	*
4295	* @returns VBox strict status code.
4296	* @param pVCpu The cross context virtual CPU structure.
4297	* @param HCPhysVmcb The host physical address of the VMCB.
4298	*
4299	* @remarks No-long-jump zone!!!
4300	*/
4301	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4302	{
4303	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4304	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4305	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4306	}
4307
4308
4309	/**
4310	* Performs some essential restoration of state after running guest (or
4311	* nested-guest) code in AMD-V.
4312	*
4313	* @param pVCpu The cross context virtual CPU structure.
4314	* @param pSvmTransient Pointer to the SVM transient structure.
4315	* @param rcVMRun Return code of VMRUN.
4316	*
4317	* @remarks Called with interrupts disabled.
4318	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4319	* unconditionally when it is safe to do so.
4320	*/
4321	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4322	{
4323	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4324
4325	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4326	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4327
4328	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4329	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4330
4331	/* TSC read must be done early for maximum accuracy. */
4332	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4333	{
4334	if (!pSvmTransient->fIsNestedGuest)
4335	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4336	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4337	else
4338	{
4339	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4340	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4341	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4342	}
4343	#endif
4344	}
4345
4346	if (pSvmTransient->fRestoreTscAuxMsr)
4347	{
4348	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4349	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4350	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4351	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4352	}
4353
4354	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4355	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4356	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4357	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4358
4359	Assert(!(ASMGetFlags() & X86_EFL_IF));
4360	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4361	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4362
4363	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4364	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4365	{
4366	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4367	return;
4368	}
4369
4370	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4371	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4372	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4373	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4374
4375	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4376	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4377	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4378	#else
4379	/*
4380	* Always import the following:
4381	*
4382	* - RIP for exit optimizations and evaluating event injection on re-entry.
4383	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4384	* state on preemption.
4385	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4386	* - CS for exit optimizations.
4387	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4388	* assembly switcher code.
4389	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4390	*/
4391	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4392	\| CPUMCTX_EXTRN_RFLAGS
4393	\| CPUMCTX_EXTRN_RAX
4394	\| CPUMCTX_EXTRN_RSP
4395	\| CPUMCTX_EXTRN_CS
4396	\| CPUMCTX_EXTRN_HWVIRT
4397	\| CPUMCTX_EXTRN_INHIBIT_INT
4398	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4399	\| HMSVM_CPUMCTX_SHARED_STATE);
4400	#endif
4401
4402	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4403	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4404	{
4405	Assert(!pSvmTransient->fIsNestedGuest);
4406	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4407	if ( pVM->hm.s.fTprPatchingActive
4408	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4409	{
4410	int rc = PDMApicSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4411	AssertRC(rc);
4412	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4413	}
4414	/* Sync TPR when we aren't intercepting CR8 writes. */
4415	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4416	{
4417	int rc = PDMApicSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4418	AssertRC(rc);
4419	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4420	}
4421	}
4422
4423	#ifdef DEBUG_ramshankar
4424	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4425	{
4426	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4427	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4428	0 /* uVerbose */);
4429	}
4430	#endif
4431
4432	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4433	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4434	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4435	}
4436
4437
4438	/**
4439	* Runs the guest code using AMD-V.
4440	*
4441	* @returns Strict VBox status code.
4442	* @param pVCpu The cross context virtual CPU structure.
4443	* @param pcLoops Pointer to the number of executed loops.
4444	*/
4445	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4446	{
4447	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4448	Assert(pcLoops);
4449	Assert(*pcLoops <= cMaxResumeLoops);
4450
4451	SVMTRANSIENT SvmTransient;
4452	RT_ZERO(SvmTransient);
4453	SvmTransient.fUpdateTscOffsetting = true;
4454	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4455
4456	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4457	for (;;)
4458	{
4459	Assert(!HMR0SuspendPending());
4460	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4461
4462	/* Preparatory work for running nested-guest code, this may force us to return to
4463	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4464	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4465	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4466	if (rc != VINF_SUCCESS)
4467	break;
4468
4469	/*
4470	* No longjmps to ring-3 from this point on!!!
4471	*
4472	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4473	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4474	*/
4475	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4476	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4477
4478	/* Restore any residual host-state and save any bits shared between host and guest
4479	into the guest-CPU state. Re-enables interrupts! */
4480	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4481
4482	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4483	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4484	{
4485	if (rc == VINF_SUCCESS)
4486	rc = VERR_SVM_INVALID_GUEST_STATE;
4487	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4488	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4489	break;
4490	}
4491
4492	/* Handle the #VMEXIT. */
4493	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4494	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4495	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4496	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4497	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4498	if (rc != VINF_SUCCESS)
4499	break;
4500	if (++(*pcLoops) >= cMaxResumeLoops)
4501	{
4502	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4503	rc = VINF_EM_RAW_INTERRUPT;
4504	break;
4505	}
4506	}
4507
4508	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4509	return rc;
4510	}
4511
4512
4513	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4514	/**
4515	* Runs the nested-guest code using AMD-V.
4516	*
4517	* @returns Strict VBox status code.
4518	* @param pVCpu The cross context virtual CPU structure.
4519	* @param pcLoops Pointer to the number of executed loops. If we're switching
4520	* from the guest-code execution loop to this nested-guest
4521	* execution loop pass the remainder value, else pass 0.
4522	*/
4523	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4524	{
4525	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4526	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4527	Assert(pcLoops);
4528	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4529	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4530	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4531
4532	SVMTRANSIENT SvmTransient;
4533	RT_ZERO(SvmTransient);
4534	SvmTransient.fUpdateTscOffsetting = true;
4535	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4536	SvmTransient.fIsNestedGuest = true;
4537
4538	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4539	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4540
4541	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4542	for (;;)
4543	{
4544	Assert(!HMR0SuspendPending());
4545	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4546
4547	/* Preparatory work for running nested-guest code, this may force us to return to
4548	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4549	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4550	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4551	if ( rc != VINF_SUCCESS
4552	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4553	break;
4554
4555	/*
4556	* No longjmps to ring-3 from this point on!!!
4557	*
4558	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4559	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4560	*/
4561	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4562
4563	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4564
4565	/* Restore any residual host-state and save any bits shared between host and guest
4566	into the guest-CPU state. Re-enables interrupts! */
4567	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4568
4569	if (RT_LIKELY( rc == VINF_SUCCESS
4570	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4571	{ /* extremely likely */ }
4572	else
4573	{
4574	/* VMRUN failed, shouldn't really happen, Guru. */
4575	if (rc != VINF_SUCCESS)
4576	break;
4577
4578	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4579	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4580	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4581	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4582	break;
4583	}
4584
4585	/* Handle the #VMEXIT. */
4586	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4587	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4588	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4589	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4590	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4591	if (rc == VINF_SUCCESS)
4592	{
4593	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4594	{
4595	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4596	rc = VINF_SVM_VMEXIT;
4597	}
4598	else
4599	{
4600	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4601	continue;
4602	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4603	rc = VINF_EM_RAW_INTERRUPT;
4604	}
4605	}
4606	else
4607	Assert(rc != VINF_SVM_VMEXIT);
4608	break;
4609	/** @todo NSTSVM: handle single-stepping. */
4610	}
4611
4612	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4613	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4614
4615	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4616	return rc;
4617	}
4618	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4619
4620
4621	/**
4622	* Checks if any expensive dtrace probes are enabled and we should go to the
4623	* debug loop.
4624	*
4625	* @returns true if we should use debug loop, false if not.
4626	*/
4627	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4628	{
4629	/* It's probably faster to OR the raw 32-bit counter variables together.
4630	Since the variables are in an array and the probes are next to one
4631	another (more or less), we have good locality. So, better read
4632	eight-nine cache lines ever time and only have one conditional, than
4633	128+ conditionals, right? */
4634	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4635	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4636	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4637	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4638	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4639	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4640	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4641	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4642	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4643	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4644	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4645	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4646	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4647	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4648	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4649	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4650	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4651	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4652	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4653	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4654	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4655	) != 0
4656	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4657	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4658	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4659	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4660	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4661	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4662	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4663	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4664	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4665	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4666	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4673	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4675	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4676	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4677	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4678	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4679	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4680	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4681	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4682	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4683	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4684	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4685	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4686	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4687	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4689	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4690	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4691	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4692	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4693	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4694	) != 0
4695	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4696	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4697	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4698	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4699	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4700	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4701	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4702	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4703	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4704	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4705	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4713	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4715	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4716	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4717	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4718	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4719	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4720	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4721	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4722	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4723	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4724	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4725	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4726	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4727	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4729	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4730	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4731	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4732	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4733	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4734	) != 0;
4735	}
4736
4737
4738	/**
4739	* Runs the guest code using AMD-V.
4740	*
4741	* @returns Strict VBox status code.
4742	* @param pVCpu The cross context virtual CPU structure.
4743	*/
4744	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4745	{
4746	AssertPtr(pVCpu);
4747	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4748	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4749	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4750	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4751
4752	uint32_t cLoops = 0;
4753	VBOXSTRICTRC rc;
4754	for (;;)
4755	{
4756	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4757	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4758	#else
4759	NOREF(pCtx);
4760	bool const fInNestedGuestMode = false;
4761	#endif
4762	if (!fInNestedGuestMode)
4763	{
4764	if ( !pVCpu->hm.s.fUseDebugLoop
4765	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4766	&& !DBGFIsStepping(pVCpu)
4767	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledSwBreakpoints)
4768	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4769	else
4770	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4771	}
4772	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4773	else
4774	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4775
4776	if (rc == VINF_SVM_VMRUN)
4777	{
4778	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4779	continue;
4780	}
4781	if (rc == VINF_SVM_VMEXIT)
4782	{
4783	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4784	continue;
4785	}
4786	#endif
4787	break;
4788	}
4789
4790	/* Fixup error codes. */
4791	if (rc == VERR_EM_INTERPRETER)
4792	rc = VINF_EM_RAW_EMULATE_INSTR;
4793	else if (rc == VINF_EM_RESET)
4794	rc = VINF_EM_TRIPLE_FAULT;
4795
4796	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4797	rc = hmR0SvmExitToRing3(pVCpu, rc);
4798	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4799	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4800	return rc;
4801	}
4802
4803	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4804
4805	/**
4806	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4807	*
4808	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4809	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4810	*/
4811	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4812	{
4813	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4814	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4815	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4816	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4817	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4818	const bool fRep = pIoExitInfo->n.u1Rep;
4819	const bool fStrIo = pIoExitInfo->n.u1Str;
4820
4821	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4822	NULL /* pIoExitInfo */);
4823	}
4824
4825
4826	/**
4827	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4828	* SVM_EXIT_INVALID).
4829	*
4830	* @returns VBox status code (informational status codes included).
4831	* @param pVCpu The cross context virtual CPU structure.
4832	* @param pSvmTransient Pointer to the SVM transient structure.
4833	*/
4834	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4835	{
4836	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4837	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4838	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4839
4840	/*
4841	* We import the complete state here because we use separate VMCBs for the guest and the
4842	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4843	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4844	*/
4845	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4846	do { \
4847	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4848	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4849	} while (0)
4850
4851	/*
4852	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4853	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4854	*/
4855	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4856	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4857	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4858	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4859	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4860	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4861
4862	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4863	switch (uExitCode)
4864	{
4865	case SVM_EXIT_CPUID:
4866	{
4867	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4868	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4869	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4870	}
4871
4872	case SVM_EXIT_RDTSC:
4873	{
4874	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4875	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4876	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4877	}
4878
4879	case SVM_EXIT_RDTSCP:
4880	{
4881	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4882	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4883	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4884	}
4885
4886	case SVM_EXIT_MONITOR:
4887	{
4888	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4889	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4890	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4891	}
4892
4893	case SVM_EXIT_MWAIT:
4894	{
4895	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4896	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4897	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4898	}
4899
4900	case SVM_EXIT_HLT:
4901	{
4902	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4903	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4904	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4905	}
4906
4907	case SVM_EXIT_MSR:
4908	{
4909	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4910	{
4911	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4912	uint16_t offMsrpm;
4913	uint8_t uMsrpmBit;
4914	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4915	if (RT_SUCCESS(rc))
4916	{
4917	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4918	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4919
4920	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4921	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4922	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4923
4924	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4925	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4926	{
4927	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4928	}
4929	}
4930	else
4931	{
4932	/*
4933	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4934	* See AMD-V spec. "15.11 MSR Intercepts".
4935	*/
4936	Assert(rc == VERR_OUT_OF_RANGE);
4937	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4938	}
4939	}
4940	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4941	}
4942
4943	case SVM_EXIT_IOIO:
4944	{
4945	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4946	{
4947	SVMIOIOEXITINFO IoExitInfo;
4948	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4949	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4950	if (fIntercept)
4951	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4952	}
4953	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4954	}
4955
4956	case SVM_EXIT_XCPT_PF:
4957	{
4958	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4959	if (pVM->hmr0.s.fNestedPaging)
4960	{
4961	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4962	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4963
4964	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4965	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4966	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4967
4968	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4969	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4970	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4971	return VINF_SUCCESS;
4972	}
4973	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4974	}
4975
4976	case SVM_EXIT_XCPT_UD:
4977	{
4978	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4979	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4980	hmR0SvmSetPendingXcptUD(pVCpu);
4981	return VINF_SUCCESS;
4982	}
4983
4984	case SVM_EXIT_XCPT_MF:
4985	{
4986	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4987	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4988	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4989	}
4990
4991	case SVM_EXIT_XCPT_DB:
4992	{
4993	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4994	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4995	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
4996	}
4997
4998	case SVM_EXIT_XCPT_AC:
4999	{
5000	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5001	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5002	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5003	}
5004
5005	case SVM_EXIT_XCPT_BP:
5006	{
5007	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5008	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5009	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5010	}
5011
5012	case SVM_EXIT_READ_CR0:
5013	case SVM_EXIT_READ_CR3:
5014	case SVM_EXIT_READ_CR4:
5015	{
5016	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5017	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5018	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5019	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5020	}
5021
5022	case SVM_EXIT_CR0_SEL_WRITE:
5023	{
5024	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5025	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5026	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5027	}
5028
5029	case SVM_EXIT_WRITE_CR0:
5030	case SVM_EXIT_WRITE_CR3:
5031	case SVM_EXIT_WRITE_CR4:
5032	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5033	{
5034	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5035	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5036
5037	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5038	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5039	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5040	}
5041
5042	case SVM_EXIT_PAUSE:
5043	{
5044	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5045	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5046	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5047	}
5048
5049	case SVM_EXIT_VINTR:
5050	{
5051	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5052	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5053	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5054	}
5055
5056	case SVM_EXIT_INTR:
5057	case SVM_EXIT_NMI:
5058	case SVM_EXIT_SMI:
5059	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5060	{
5061	/*
5062	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5063	*
5064	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5065	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5066	* nested-guest #VMEXIT.
5067	*
5068	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5069	* while trying to inject interrupts, see comment at the top of this function.
5070	*/
5071	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5072	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5073	}
5074
5075	case SVM_EXIT_FERR_FREEZE:
5076	{
5077	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5078	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5079	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5080	}
5081
5082	case SVM_EXIT_INVLPG:
5083	{
5084	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5085	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5086	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5087	}
5088
5089	case SVM_EXIT_WBINVD:
5090	{
5091	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5092	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5093	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5094	}
5095
5096	case SVM_EXIT_INVD:
5097	{
5098	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5099	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5100	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5101	}
5102
5103	case SVM_EXIT_RDPMC:
5104	{
5105	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5106	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5107	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5108	}
5109
5110	default:
5111	{
5112	switch (uExitCode)
5113	{
5114	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5115	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5116	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5117	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5118	{
5119	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5120	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5121	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5122	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5123	}
5124
5125	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5126	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5127	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5128	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5129	{
5130	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5131	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5132	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5133	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5134	}
5135
5136	case SVM_EXIT_XCPT_DE:
5137	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5138	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5139	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5140	case SVM_EXIT_XCPT_OF:
5141	case SVM_EXIT_XCPT_BR:
5142	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5143	case SVM_EXIT_XCPT_NM:
5144	case SVM_EXIT_XCPT_DF:
5145	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5146	case SVM_EXIT_XCPT_TS:
5147	case SVM_EXIT_XCPT_NP:
5148	case SVM_EXIT_XCPT_SS:
5149	case SVM_EXIT_XCPT_GP:
5150	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5151	case SVM_EXIT_XCPT_15: /* Reserved. */
5152	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5153	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5154	case SVM_EXIT_XCPT_MC:
5155	case SVM_EXIT_XCPT_XF:
5156	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5157	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5158	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5159	{
5160	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5161	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5162	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5163	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5164	}
5165
5166	case SVM_EXIT_XSETBV:
5167	{
5168	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5169	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5170	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5171	}
5172
5173	case SVM_EXIT_TASK_SWITCH:
5174	{
5175	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5176	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5177	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5178	}
5179
5180	case SVM_EXIT_IRET:
5181	{
5182	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5183	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5184	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5185	}
5186
5187	case SVM_EXIT_SHUTDOWN:
5188	{
5189	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5190	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5191	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5192	}
5193
5194	case SVM_EXIT_VMMCALL:
5195	{
5196	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5197	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5198	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5199	}
5200
5201	case SVM_EXIT_CLGI:
5202	{
5203	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5204	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5205	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5206	}
5207
5208	case SVM_EXIT_STGI:
5209	{
5210	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5211	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5212	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5213	}
5214
5215	case SVM_EXIT_VMLOAD:
5216	{
5217	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5218	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5219	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5220	}
5221
5222	case SVM_EXIT_VMSAVE:
5223	{
5224	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))