HMSVMR0.cpp

Last change on this file was 106362, checked in by vboxsync, 7 weeks ago
VMM/DBGF: Prepare DBGF to support ARMv8/A64 style breakpoints for the VMM debugger. This converts the x86 centric int3 naming to software breakpoint, bugref:10393
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 380.7 KB

Line
1	/* $Id: HMSVMR0.cpp 106362 2024-10-16 13:08:09Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_HM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/thread.h>
36
37	#include <VBox/vmm/pdmapi.h>
38	#include <VBox/vmm/dbgf.h>
39	#include <VBox/vmm/iem.h>
40	#include <VBox/vmm/iom.h>
41	#include <VBox/vmm/tm.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/gcm.h>
44	#include <VBox/vmm/gim.h>
45	#include <VBox/vmm/apic.h>
46	#include "HMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include <VBox/err.h>
49	#include "HMSVMR0.h"
50	#include "dtrace/VBoxVMM.h"
51
52	#ifdef DEBUG_ramshankar
53	# define HMSVM_SYNC_FULL_GUEST_STATE
54	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
55	# define HMSVM_ALWAYS_TRAP_PF
56	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
57	#endif
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	#ifdef VBOX_WITH_STATISTICS
64	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
66	if ((u64ExitCode) == SVM_EXIT_NPF) \
67	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
68	else \
69	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
70	} while (0)
71
72	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
74	if ((u64ExitCode) == SVM_EXIT_NPF) \
75	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
76	else \
77	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
78	} while (0)
79
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
81	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
82	if ((u64ExitCode) == SVM_EXIT_NPF) \
83	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
84	else \
85	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
86	} while (0)
87	#else
88	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
89	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
90	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
91	#endif /* !VBOX_WITH_STATISTICS */
92
93	/** If we decide to use a function table approach this can be useful to
94	* switch to a "static DECLCALLBACK(int)". */
95	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
96
97	/**
98	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
99	* guest using hardware-assisted SVM.
100	*
101	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
102	* are swapped and restored across the world-switch and also registers like
103	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
104	* \#VMEXIT.
105	*/
106	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
107	\| CPUMCTX_EXTRN_RFLAGS \
108	\| CPUMCTX_EXTRN_RAX \
109	\| CPUMCTX_EXTRN_RSP \
110	\| CPUMCTX_EXTRN_SREG_MASK \
111	\| CPUMCTX_EXTRN_CR0 \
112	\| CPUMCTX_EXTRN_CR2 \
113	\| CPUMCTX_EXTRN_CR3 \
114	\| CPUMCTX_EXTRN_TABLE_MASK \
115	\| CPUMCTX_EXTRN_DR6 \
116	\| CPUMCTX_EXTRN_DR7 \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_HWVIRT \
121	\| CPUMCTX_EXTRN_INHIBIT_INT \
122	\| CPUMCTX_EXTRN_HM_SVM_MASK)
123
124	/**
125	* Subset of the guest-CPU state that is shared between the guest and host.
126	*/
127	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
128
129	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
130	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
131	do { \
132	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
133	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
134	} while (0)
135
136	/** Assert that the required state bits are fetched. */
137	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
151
152	/** Assert that we're not executing a nested-guest. */
153	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
154	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
155	#else
156	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
157	#endif
158
159	/** Assert that we're executing a nested-guest. */
160	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
161	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
162	#else
163	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
164	#endif
165
166	/** Macro for checking and returning from the using function for
167	* \#VMEXIT intercepts that maybe caused during delivering of another
168	* event in the guest. */
169	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
170	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
171	do \
172	{ \
173	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
174	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
175	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
176	else if ( rc == VINF_EM_RESET \
177	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
178	{ \
179	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
180	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
181	} \
182	else \
183	return rc; \
184	} while (0)
185	#else
186	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
187	do \
188	{ \
189	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
190	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
191	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
192	else \
193	return rc; \
194	} while (0)
195	#endif
196
197	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
198	* instruction that exited. */
199	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
200	do { \
201	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
202	(a_rc) = VINF_EM_DBG_STEPPED; \
203	} while (0)
204
205	/** Validate segment descriptor granularity bit. */
206	#ifdef VBOX_STRICT
207	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
208	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
209	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
210	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
211	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
212	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
213	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
214	#else
215	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
216	#endif
217
218	/**
219	* Exception bitmap mask for all contributory exceptions.
220	*
221	* Page fault is deliberately excluded here as it's conditional as to whether
222	* it's contributory or benign. Page faults are handled separately.
223	*/
224	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
225	\| RT_BIT(X86_XCPT_DE))
226
227	/**
228	* Mandatory/unconditional guest control intercepts.
229	*
230	* SMIs can and do happen in normal operation. We need not intercept them
231	* while executing the guest (or nested-guest).
232	*/
233	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
234	\| SVM_CTRL_INTERCEPT_NMI \
235	\| SVM_CTRL_INTERCEPT_INIT \
236	\| SVM_CTRL_INTERCEPT_RDPMC \
237	\| SVM_CTRL_INTERCEPT_CPUID \
238	\| SVM_CTRL_INTERCEPT_RSM \
239	\| SVM_CTRL_INTERCEPT_HLT \
240	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
241	\| SVM_CTRL_INTERCEPT_MSR_PROT \
242	\| SVM_CTRL_INTERCEPT_INVLPGA \
243	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
244	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
245	\| SVM_CTRL_INTERCEPT_VMRUN \
246	\| SVM_CTRL_INTERCEPT_SKINIT \
247	\| SVM_CTRL_INTERCEPT_WBINVD \
248	\| SVM_CTRL_INTERCEPT_MONITOR \
249	\| SVM_CTRL_INTERCEPT_MWAIT \
250	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
251	\| SVM_CTRL_INTERCEPT_XSETBV)
252
253	/** @name VMCB Clean Bits.
254	*
255	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
256	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
257	* memory.
258	*
259	* @{ */
260	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
261	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
262	/** I/O permission bitmap, MSR permission bitmap. */
263	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
264	/** ASID. */
265	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
266	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
267	V_INTR_VECTOR. */
268	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
269	/** Nested Paging: Nested CR3 (nCR3), PAT. */
270	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
271	/** Control registers (CR0, CR3, CR4, EFER). */
272	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
273	/** Debug registers (DR6, DR7). */
274	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
275	/** GDT, IDT limit and base. */
276	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
277	/** Segment register: CS, SS, DS, ES limit and base. */
278	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
279	/** CR2.*/
280	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
281	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
282	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
283	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
284	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
285	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
286	/** Mask of all valid VMCB Clean bits. */
287	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
288	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
289	\| HMSVM_VMCB_CLEAN_ASID \
290	\| HMSVM_VMCB_CLEAN_INT_CTRL \
291	\| HMSVM_VMCB_CLEAN_NP \
292	\| HMSVM_VMCB_CLEAN_CRX_EFER \
293	\| HMSVM_VMCB_CLEAN_DRX \
294	\| HMSVM_VMCB_CLEAN_DT \
295	\| HMSVM_VMCB_CLEAN_SEG \
296	\| HMSVM_VMCB_CLEAN_CR2 \
297	\| HMSVM_VMCB_CLEAN_LBR \
298	\| HMSVM_VMCB_CLEAN_AVIC)
299	/** @} */
300
301	/**
302	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
303	*/
304	typedef enum SVMMSREXITREAD
305	{
306	/** Reading this MSR causes a \#VMEXIT. */
307	SVMMSREXIT_INTERCEPT_READ = 0xb,
308	/** Reading this MSR does not cause a \#VMEXIT. */
309	SVMMSREXIT_PASSTHRU_READ
310	} SVMMSREXITREAD;
311
312	/**
313	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
314	*/
315	typedef enum SVMMSREXITWRITE
316	{
317	/** Writing to this MSR causes a \#VMEXIT. */
318	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a \#VMEXIT. */
320	SVMMSREXIT_PASSTHRU_WRITE
321	} SVMMSREXITWRITE;
322
323	/**
324	* SVM \#VMEXIT handler.
325	*
326	* @returns Strict VBox status code.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pSvmTransient Pointer to the SVM-transient structure.
329	*/
330	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
331
332
333	/*********************************************************************************************************************************
334	* Internal Functions *
335	*********************************************************************************************************************************/
336	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
337	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
338
339
340	/** @name \#VMEXIT handlers.
341	* @{
342	*/
343	static FNSVMEXITHANDLER hmR0SvmExitIntr;
344	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
345	static FNSVMEXITHANDLER hmR0SvmExitInvd;
346	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
347	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
348	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
349	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
350	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
351	static FNSVMEXITHANDLER hmR0SvmExitHlt;
352	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
353	static FNSVMEXITHANDLER hmR0SvmExitMwait;
354	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
355	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
356	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
357	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
358	static FNSVMEXITHANDLER hmR0SvmExitMsr;
359	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
360	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
361	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
362	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
363	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
364	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
365	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
366	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
367	static FNSVMEXITHANDLER hmR0SvmExitPause;
368	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
369	static FNSVMEXITHANDLER hmR0SvmExitIret;
370	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
371	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
372	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
373	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
374	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
375	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
376	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
377	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
378	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
379	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
380	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
381	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
382	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
383	static FNSVMEXITHANDLER hmR0SvmExitClgi;
384	static FNSVMEXITHANDLER hmR0SvmExitStgi;
385	static FNSVMEXITHANDLER hmR0SvmExitVmload;
386	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
387	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
388	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
389	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
390	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
391	#endif
392	/** @} */
393
394	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
395	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
396	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
397	#endif
398	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
399
400
401	/*********************************************************************************************************************************
402	* Global Variables *
403	*********************************************************************************************************************************/
404	/** Ring-0 memory object for the IO bitmap. */
405	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
406	/** Physical address of the IO bitmap. */
407	static RTHCPHYS g_HCPhysIOBitmap;
408	/** Pointer to the IO bitmap. */
409	static R0PTRTYPE(void *) g_pvIOBitmap;
410
411	#ifdef VBOX_STRICT
412	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
413	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
414	# define HMSVM_LOG_CS RT_BIT_32(2)
415	# define HMSVM_LOG_SS RT_BIT_32(3)
416	# define HMSVM_LOG_FS RT_BIT_32(4)
417	# define HMSVM_LOG_GS RT_BIT_32(5)
418	# define HMSVM_LOG_LBR RT_BIT_32(6)
419	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
420	\| HMSVM_LOG_CR_REGS \
421	\| HMSVM_LOG_CS \
422	\| HMSVM_LOG_SS \
423	\| HMSVM_LOG_FS \
424	\| HMSVM_LOG_GS \
425	\| HMSVM_LOG_LBR)
426
427	/**
428	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
429	*
430	* @param pVCpu The cross context virtual CPU structure.
431	* @param pVmcb Pointer to the VM control block.
432	* @param pszPrefix Log prefix.
433	* @param fFlags Log flags, see HMSVM_LOG_XXX.
434	* @param uVerbose The verbosity level, currently unused.
435	*/
436	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
437	{
438	RT_NOREF2(pVCpu, uVerbose);
439	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
440
441	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
442	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
443
444	if (fFlags & HMSVM_LOG_RBP_RSP)
445	{
446	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
447	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
448	}
449
450	if (fFlags & HMSVM_LOG_CR_REGS)
451	{
452	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
453	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
454	}
455
456	if (fFlags & HMSVM_LOG_CS)
457	{
458	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
459	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
460	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
461	}
462	if (fFlags & HMSVM_LOG_SS)
463	{
464	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
465	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
466	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
467	}
468	if (fFlags & HMSVM_LOG_FS)
469	{
470	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
471	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
472	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
473	}
474	if (fFlags & HMSVM_LOG_GS)
475	{
476	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
477	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
478	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
479	}
480
481	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
482	if (fFlags & HMSVM_LOG_LBR)
483	{
484	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
485	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
486	}
487	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
488	}
489	#endif /* VBOX_STRICT */
490
491
492	/**
493	* Sets up and activates AMD-V on the current CPU.
494	*
495	* @returns VBox status code.
496	* @param pHostCpu The HM physical-CPU structure.
497	* @param pVM The cross context VM structure. Can be
498	* NULL after a resume!
499	* @param pvCpuPage Pointer to the global CPU page.
500	* @param HCPhysCpuPage Physical address of the global CPU page.
501	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
502	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
503	* unused).
504	*/
505	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
506	PCSUPHWVIRTMSRS pHwvirtMsrs)
507	{
508	Assert(!fEnabledByHost);
509	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
510	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
511	Assert(pvCpuPage); NOREF(pvCpuPage);
512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
513
514	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
515
516	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
517	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
518
519	/*
520	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
521	*/
522	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
523	if (u64HostEfer & MSR_K6_EFER_SVME)
524	{
525	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
526	if ( pVM
527	&& pVM->hm.s.svm.fIgnoreInUseError)
528	pHostCpu->fIgnoreAMDVInUseError = true;
529
530	if (!pHostCpu->fIgnoreAMDVInUseError)
531	{
532	ASMSetFlags(fEFlags);
533	return VERR_SVM_IN_USE;
534	}
535	}
536
537	/* Turn on AMD-V in the EFER MSR. */
538	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
539
540	/* Write the physical page address where the CPU will store the host state while executing the VM. */
541	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
542
543	/* Restore interrupts. */
544	ASMSetFlags(fEFlags);
545
546	/*
547	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
548	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
549	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
550	* entirely with before executing any guest code.
551	*/
552	pHostCpu->fFlushAsidBeforeUse = true;
553
554	/*
555	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
556	*/
557	++pHostCpu->cTlbFlushes;
558
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Deactivates AMD-V on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pHostCpu The HM physical-CPU structure.
568	* @param pvCpuPage Pointer to the global CPU page.
569	* @param HCPhysCpuPage Physical address of the global CPU page.
570	*/
571	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
572	{
573	RT_NOREF1(pHostCpu);
574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
575	AssertReturn( HCPhysCpuPage
576	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
577	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
578
579	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
580	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
581
582	/* Turn off AMD-V in the EFER MSR. */
583	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
584	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
585
586	/* Invalidate host state physical address. */
587	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
588
589	/* Restore interrupts. */
590	ASMSetFlags(fEFlags);
591
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Does global AMD-V initialization (called during module initialization).
598	*
599	* @returns VBox status code.
600	*/
601	VMMR0DECL(int) SVMR0GlobalInit(void)
602	{
603	/*
604	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
605	* intercept all IO accesses, it's done once globally here instead of per-VM.
606	*/
607	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
608	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT,
609	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
610	if (RT_FAILURE(rc))
611	return rc;
612
613	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
614	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
615
616	/* Set all bits to intercept all IO accesses. */
617	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
618
619	return VINF_SUCCESS;
620	}
621
622
623	/**
624	* Does global AMD-V termination (called during module termination).
625	*/
626	VMMR0DECL(void) SVMR0GlobalTerm(void)
627	{
628	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
629	{
630	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
631	g_pvIOBitmap = NULL;
632	g_HCPhysIOBitmap = 0;
633	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
634	}
635	}
636
637
638	/**
639	* Frees any allocated per-VCPU structures for a VM.
640	*
641	* @param pVM The cross context VM structure.
642	*/
643	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
644	{
645	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
646	{
647	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
648	AssertPtr(pVCpu);
649
650	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
651	{
652	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
653	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
654	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
655	}
656
657	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
658	{
659	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
660	pVCpu->hmr0.s.svm.pVmcb = NULL;
661	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
662	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
663	}
664
665	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
666	{
667	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
668	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
669	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
670	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
671	}
672	}
673	}
674
675
676	/**
677	* Sets pfnVMRun to the best suited variant.
678	*
679	* This must be called whenever anything changes relative to the SVMR0VMRun
680	* variant selection:
681	* - pVCpu->hm.s.fLoadSaveGuestXcr0
682	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
683	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
684	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
685	* - Perhaps: CPUMCTX.fXStateMask (windows only)
686	*
687	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
688	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
689	*/
690	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
691	{
692	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
693	{
694	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
695	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
696	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
697	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
698	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
699	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
700	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
701	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
702	};
703	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
704	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
705	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
706	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
707	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
708	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
709	}
710
711
712	/**
713	* Selector FNHMSVMVMRUN implementation.
714	*/
715	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
716	{
717	hmR0SvmUpdateVmRunFunction(pVCpu);
718	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
719	}
720
721
722	/**
723	* Does per-VM AMD-V initialization.
724	*
725	* @returns VBox status code.
726	* @param pVM The cross context VM structure.
727	*/
728	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
729	{
730	int rc = VERR_INTERNAL_ERROR_5;
731
732	/*
733	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
734	*/
735	uint32_t u32Family;
736	uint32_t u32Model;
737	uint32_t u32Stepping;
738	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
739	{
740	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
741	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
742	}
743
744	/*
745	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
746	*/
747	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
748	{
749	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
750	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
751	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
752	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
753	}
754
755	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
756	{
757	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
758
759	/*
760	* Initialize the hardware-assisted SVM guest-execution handler.
761	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
762	*/
763	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
764
765	/*
766	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
767	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
768	*/
769	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
770	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
771	if (RT_FAILURE(rc))
772	goto failure_cleanup;
773
774	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
775	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
776	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
777
778	/*
779	* Allocate one page for the guest-state VMCB.
780	*/
781	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
782	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
783	if (RT_FAILURE(rc))
784	goto failure_cleanup;
785
786	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
787	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
788	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
789
790	/*
791	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
792	* SVM to not require one.
793	*/
794	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
795	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
796	if (RT_FAILURE(rc))
797	goto failure_cleanup;
798
799	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
800	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
801	/* Set all bits to intercept all MSR accesses (changed later on). */
802	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
803	}
804
805	return VINF_SUCCESS;
806
807	failure_cleanup:
808	hmR0SvmFreeStructs(pVM);
809	return rc;
810	}
811
812
813	/**
814	* Does per-VM AMD-V termination.
815	*
816	* @returns VBox status code.
817	* @param pVM The cross context VM structure.
818	*/
819	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
820	{
821	hmR0SvmFreeStructs(pVM);
822	return VINF_SUCCESS;
823	}
824
825
826	/**
827	* Returns whether the VMCB Clean Bits feature is supported.
828	*
829	* @returns @c true if supported, @c false otherwise.
830	* @param pVCpu The cross context virtual CPU structure.
831	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
832	*/
833	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
834	{
835	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
836	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
837	if (!fIsNestedGuest)
838	return fHostVmcbCleanBits;
839	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
840	}
841
842
843	/**
844	* Returns whether the decode assists feature is supported.
845	*
846	* @returns @c true if supported, @c false otherwise.
847	* @param pVCpu The cross context virtual CPU structure.
848	*/
849	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
850	{
851	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
852	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
853	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
854	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
855	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
856	#endif
857	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
858	}
859
860
861	/**
862	* Returns whether the NRIP_SAVE feature is supported.
863	*
864	* @returns @c true if supported, @c false otherwise.
865	* @param pVCpu The cross context virtual CPU structure.
866	*/
867	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
868	{
869	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
870	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
871	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
872	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
873	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
874	#endif
875	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
876	}
877
878
879	/**
880	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
881	*
882	* @param pVCpu The cross context virtual CPU structure.
883	* @param pbMsrBitmap Pointer to the MSR bitmap.
884	* @param idMsr The MSR for which the permissions are being set.
885	* @param enmRead MSR read permissions.
886	* @param enmWrite MSR write permissions.
887	*
888	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
889	* caller needs to take care of this.
890	*/
891	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
892	SVMMSREXITWRITE enmWrite)
893	{
894	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
895	uint16_t offMsrpm;
896	uint8_t uMsrpmBit;
897	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
898	AssertRC(rc);
899
900	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
901	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
902
903	pbMsrBitmap += offMsrpm;
904	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
905	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
906	else
907	{
908	if (!fInNestedGuestMode)
909	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
910	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
911	else
912	{
913	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
914	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
915	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
916	else
917	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
918	}
919	#endif
920	}
921
922	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
923	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
924	else
925	{
926	if (!fInNestedGuestMode)
927	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
928	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
929	else
930	{
931	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
932	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
933	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
934	else
935	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
936	}
937	#endif
938	}
939	}
940
941
942	/**
943	* Sets up AMD-V for the specified VM.
944	* This function is only called once per-VM during initalization.
945	*
946	* @returns VBox status code.
947	* @param pVM The cross context VM structure.
948	*/
949	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
950	{
951	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
952	AssertReturn(pVM, VERR_INVALID_PARAMETER);
953
954	/*
955	* Validate and copy over some parameters.
956	*/
957	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
958	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
959	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
960	pVM->hmr0.s.fNestedPaging = fNestedPaging;
961	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
962
963	/*
964	* Determin some configuration parameters.
965	*/
966	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
967	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
968	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
969
970	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
971	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
972
973	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
974	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
975	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
976
977	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
978	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
979	#endif
980
981	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
982	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
983	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
984	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
985
986	/* Always trap #AC for reasons of security. */
987	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
988
989	/* Always trap #DB for reasons of security. */
990	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
991
992	/* Trap exceptions unconditionally (debug purposes). */
993	#ifdef HMSVM_ALWAYS_TRAP_PF
994	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
995	#endif
996	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
997	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
998	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
999	\| RT_BIT_32(X86_XCPT_DE)
1000	\| RT_BIT_32(X86_XCPT_NM)
1001	\| RT_BIT_32(X86_XCPT_UD)
1002	\| RT_BIT_32(X86_XCPT_NP)
1003	\| RT_BIT_32(X86_XCPT_SS)
1004	\| RT_BIT_32(X86_XCPT_GP)
1005	\| RT_BIT_32(X86_XCPT_PF)
1006	\| RT_BIT_32(X86_XCPT_MF)
1007	;
1008	#endif
1009
1010	/* Apply the exceptions intercepts needed by the GIM provider. */
1011	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1012	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1013
1014	/* Apply the exceptions intercepts needed by the GCM fixers. */
1015	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1016	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1017
1018	/* The mesa 3d driver hack needs #GP. */
1019	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1020	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1021
1022	/* Set up unconditional intercepts and conditions. */
1023	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1024	\| SVM_CTRL_INTERCEPT_VMMCALL
1025	\| SVM_CTRL_INTERCEPT_VMSAVE
1026	\| SVM_CTRL_INTERCEPT_VMLOAD
1027	\| SVM_CTRL_INTERCEPT_CLGI
1028	\| SVM_CTRL_INTERCEPT_STGI;
1029
1030	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1031	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1032	#endif
1033
1034	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1035	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1036	{
1037	/* Virtualized VMSAVE/VMLOAD. */
1038	if (fUseVirtVmsaveVmload)
1039	{
1040	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1041	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1042	\| SVM_CTRL_INTERCEPT_VMLOAD);
1043	}
1044	else
1045	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1046
1047	/* Virtual GIF. */
1048	if (fUseVGif)
1049	{
1050	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1051	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1052	\| SVM_CTRL_INTERCEPT_STGI);
1053	}
1054	else
1055	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1056	}
1057	else
1058	#endif
1059	{
1060	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1061	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1062	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1063	}
1064
1065	/* CR4 writes must always be intercepted for tracking PGM mode changes and
1066	AVX (for XCR0 syncing during worlds switching). */
1067	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1068
1069	/* Intercept all DRx reads and writes by default. Changed later on. */
1070	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1071	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1072
1073	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1074	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1075
1076	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1077	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1078	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1079
1080	/* Set the IO permission bitmap physical addresses. */
1081	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1082
1083	/* LBR virtualization. */
1084	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1085
1086	/* The host ASID MBZ, for the guest start with 1. */
1087	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1088
1089	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1090	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1091
1092	/* Without Nested Paging, we need additionally intercepts. */
1093	if (!fNestedPaging)
1094	{
1095	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1096	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1097	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1098
1099	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1100	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1101	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1102
1103	/* Page faults must be intercepted to implement shadow paging. */
1104	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1105	}
1106
1107	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1108	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1109	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1110
1111	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1112	if (fUsePauseFilter)
1113	{
1114	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1115	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1116	if (fPauseFilterThreshold)
1117	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1118	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1119	}
1120
1121	/*
1122	* Setup the MSR permission bitmap.
1123	* The following MSRs are saved/restored automatically during the world-switch.
1124	* Don't intercept guest read/write accesses to these MSRs.
1125	*/
1126	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1127	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1128	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1129	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1130	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1131	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1132	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1133	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1134	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1135	{
1136	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1137	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1138	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1139	}
1140	else
1141	{
1142	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1143	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1144	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1145	}
1146	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1147
1148	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1149	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1150
1151	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1152	{
1153	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1154	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1155	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1156	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1157
1158	/* Copy the VMCB control area. */
1159	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1160
1161	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1162	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1163	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1164	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1165
1166	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1167	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1168
1169	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1170	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1171	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1172	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1173	}
1174
1175	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1176	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1177	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1178	#else
1179	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1180	#endif
1181	return VINF_SUCCESS;
1182	}
1183
1184
1185	/**
1186	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1187	*
1188	* @returns Pointer to the current context VMCB.
1189	* @param pVCpu The cross context virtual CPU structure.
1190	*/
1191	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1192	{
1193	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1194	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1195	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1196	#endif
1197	return pVCpu->hmr0.s.svm.pVmcb;
1198	}
1199
1200
1201	/**
1202	* Gets a pointer to the nested-guest VMCB cache.
1203	*
1204	* @returns Pointer to the nested-guest VMCB cache.
1205	* @param pVCpu The cross context virtual CPU structure.
1206	*/
1207	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1208	{
1209	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1210	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1211	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1212	#else
1213	RT_NOREF(pVCpu);
1214	return NULL;
1215	#endif
1216	}
1217
1218
1219	/**
1220	* Invalidates a guest page by guest virtual address.
1221	*
1222	* @returns VBox status code.
1223	* @param pVCpu The cross context virtual CPU structure.
1224	* @param GCVirt Guest virtual address of the page to invalidate.
1225	*/
1226	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1227	{
1228	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1229
1230	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1231
1232	/* Skip it if a TLB flush is already pending. */
1233	if (!fFlushPending)
1234	{
1235	Log4Func(("%#RGv\n", GCVirt));
1236
1237	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1238	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1239
1240	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1241	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1242	}
1243	return VINF_SUCCESS;
1244	}
1245
1246
1247	/**
1248	* Flushes the appropriate tagged-TLB entries.
1249	*
1250	* @param pHostCpu The HM physical-CPU structure.
1251	* @param pVCpu The cross context virtual CPU structure.
1252	* @param pVmcb Pointer to the VM control block.
1253	*/
1254	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1255	{
1256	/*
1257	* Force a TLB flush for the first world switch if the current CPU differs from the one
1258	* we ran on last. This can happen both for start & resume due to long jumps back to
1259	* ring-3.
1260	*
1261	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1262	* correlation between it and the physical CPU.
1263	*
1264	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1265	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1266	*/
1267	bool fNewAsid = false;
1268	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1269	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1270	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1271	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1272	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1273	#endif
1274	)
1275	{
1276	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1277	pVCpu->hmr0.s.fForceTLBFlush = true;
1278	fNewAsid = true;
1279	}
1280
1281	/* Set TLB flush state as checked until we return from the world switch. */
1282	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1283
1284	/* Check for explicit TLB flushes. */
1285	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1286	{
1287	pVCpu->hmr0.s.fForceTLBFlush = true;
1288	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1289	}
1290
1291	/*
1292	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1293	* This Host CPU requirement takes precedence.
1294	*/
1295	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1296	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1297	{
1298	pHostCpu->uCurrentAsid = 1;
1299	pVCpu->hmr0.s.uCurrentAsid = 1;
1300	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1301	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1302	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1303
1304	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1305	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1306	}
1307	else
1308	{
1309	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1310	if (pVCpu->hmr0.s.fForceTLBFlush)
1311	{
1312	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1313	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1314
1315	if (fNewAsid)
1316	{
1317	++pHostCpu->uCurrentAsid;
1318
1319	bool fHitASIDLimit = false;
1320	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1321	{
1322	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1323	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1324	fHitASIDLimit = true;
1325	}
1326
1327	if ( fHitASIDLimit
1328	\|\| pHostCpu->fFlushAsidBeforeUse)
1329	{
1330	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1331	pHostCpu->fFlushAsidBeforeUse = false;
1332	}
1333
1334	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1335	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1336	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1337	}
1338	else
1339	{
1340	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1341	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1342	else
1343	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1344	}
1345
1346	pVCpu->hmr0.s.fForceTLBFlush = false;
1347	}
1348	}
1349
1350	/* Update VMCB with the ASID. */
1351	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1352	{
1353	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1354	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1355	}
1356
1357	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1358	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1359	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1360	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1361	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1362	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1363	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1364	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1365
1366	#ifdef VBOX_WITH_STATISTICS
1367	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1368	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1369	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1370	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1371	{
1372	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1373	}
1374	else
1375	{
1376	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1377	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1378	}
1379	#endif
1380	}
1381
1382
1383	/**
1384	* Sets an exception intercept in the specified VMCB.
1385	*
1386	* @param pVmcb Pointer to the VM control block.
1387	* @param uXcpt The exception (X86_XCPT_*).
1388	*/
1389	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1390	{
1391	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1392	{
1393	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1394	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1395	}
1396	}
1397
1398
1399	/**
1400	* Clears an exception intercept in the specified VMCB.
1401	*
1402	* @param pVCpu The cross context virtual CPU structure.
1403	* @param pVmcb Pointer to the VM control block.
1404	* @param uXcpt The exception (X86_XCPT_*).
1405	*
1406	* @remarks This takes into account if we're executing a nested-guest and only
1407	* removes the exception intercept if both the guest -and- nested-guest
1408	* are not intercepting it.
1409	*/
1410	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1411	{
1412	Assert(uXcpt != X86_XCPT_DB);
1413	Assert(uXcpt != X86_XCPT_AC);
1414	Assert(uXcpt != X86_XCPT_GP);
1415	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1416	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1417	{
1418	bool fRemove = true;
1419	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1420	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1421	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1422	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1423	{
1424	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1425	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1426	}
1427	# else
1428	RT_NOREF(pVCpu);
1429	# endif
1430	if (fRemove)
1431	{
1432	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1433	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1434	}
1435	}
1436	#else
1437	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1438	#endif
1439	}
1440
1441
1442	/**
1443	* Sets a control intercept in the specified VMCB.
1444	*
1445	* @param pVmcb Pointer to the VM control block.
1446	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1447	*/
1448	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1449	{
1450	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1451	{
1452	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1453	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1454	}
1455	}
1456
1457
1458	/**
1459	* Clears a control intercept in the specified VMCB.
1460	*
1461	* @returns @c true if the intercept is still set, @c false otherwise.
1462	* @param pVCpu The cross context virtual CPU structure.
1463	* @param pVmcb Pointer to the VM control block.
1464	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1465	*
1466	* @remarks This takes into account if we're executing a nested-guest and only
1467	* removes the control intercept if both the guest -and- nested-guest
1468	* are not intercepting it.
1469	*/
1470	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1471	{
1472	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1473	{
1474	bool fRemove = true;
1475	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1476	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1477	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1478	{
1479	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1480	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1481	}
1482	#else
1483	RT_NOREF(pVCpu);
1484	#endif
1485	if (fRemove)
1486	{
1487	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1488	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1489	}
1490	}
1491
1492	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1493	}
1494
1495
1496	/**
1497	* Exports the guest (or nested-guest) CR0 into the VMCB.
1498	*
1499	* @param pVCpu The cross context virtual CPU structure.
1500	* @param pVmcb Pointer to the VM control block.
1501	*
1502	* @remarks This assumes we always pre-load the guest FPU.
1503	* @remarks No-long-jump zone!!!
1504	*/
1505	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1506	{
1507	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1508
1509	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1510	uint64_t const uGuestCr0 = pCtx->cr0;
1511	uint64_t uShadowCr0 = uGuestCr0;
1512
1513	/* Always enable caching. */
1514	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1515
1516	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1517	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1518	{
1519	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1520	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1521	}
1522
1523	/*
1524	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1525	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1526	* see @bugref{7243#c103}.
1527	*/
1528	if (!(uGuestCr0 & X86_CR0_NE))
1529	{
1530	uShadowCr0 \|= X86_CR0_NE;
1531	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1532	}
1533	else
1534	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1535
1536	/*
1537	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1538	*
1539	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1540	* see @bugref{6944}.
1541	*
1542	* We also don't ever want to honor weird things like cache disable from the guest.
1543	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1544	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1545	*/
1546	if (uShadowCr0 == uGuestCr0)
1547	{
1548	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1549	{
1550	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1551	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1552	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1553	}
1554	else
1555	{
1556	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1557	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1558	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1559	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1560	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1561	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1562	}
1563	}
1564	else
1565	{
1566	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1567	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1568	}
1569	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1570
1571	Assert(!RT_HI_U32(uShadowCr0));
1572	if (pVmcb->guest.u64CR0 != uShadowCr0)
1573	{
1574	pVmcb->guest.u64CR0 = uShadowCr0;
1575	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1576	}
1577	}
1578
1579
1580	/**
1581	* Exports the guest (or nested-guest) CR3 into the VMCB.
1582	*
1583	* @param pVCpu The cross context virtual CPU structure.
1584	* @param pVmcb Pointer to the VM control block.
1585	*
1586	* @remarks No-long-jump zone!!!
1587	*/
1588	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1589	{
1590	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1591
1592	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1593	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1594	if (pVM->hmr0.s.fNestedPaging)
1595	{
1596	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1597	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1598	pVmcb->guest.u64CR3 = pCtx->cr3;
1599	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1600	}
1601	else
1602	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1603
1604	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1605	}
1606
1607
1608	/**
1609	* Exports the guest (or nested-guest) CR4 into the VMCB.
1610	*
1611	* @param pVCpu The cross context virtual CPU structure.
1612	* @param pVmcb Pointer to the VM control block.
1613	*
1614	* @remarks No-long-jump zone!!!
1615	*/
1616	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1617	{
1618	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1619
1620	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1621	uint64_t uShadowCr4 = pCtx->cr4;
1622	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1623	{
1624	switch (pVCpu->hm.s.enmShadowMode)
1625	{
1626	case PGMMODE_REAL:
1627	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1628	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1629
1630	case PGMMODE_32_BIT: /* 32-bit paging. */
1631	uShadowCr4 &= ~X86_CR4_PAE;
1632	break;
1633
1634	case PGMMODE_PAE: /* PAE paging. */
1635	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1636	/** Must use PAE paging as we could use physical memory > 4 GB */
1637	uShadowCr4 \|= X86_CR4_PAE;
1638	break;
1639
1640	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1641	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1642	#ifdef VBOX_WITH_64_BITS_GUESTS
1643	break;
1644	#else
1645	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1646	#endif
1647
1648	default: /* shut up gcc */
1649	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1650	}
1651	}
1652
1653	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1654	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1655	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1656	{
1657	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1658	hmR0SvmUpdateVmRunFunction(pVCpu);
1659	}
1660
1661	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1662	if (uShadowCr4 == pCtx->cr4)
1663	{
1664	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1665	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1666	else
1667	{
1668	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1669	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1670	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1671	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1672	}
1673	}
1674	else
1675	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1676
1677	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking
1678	PGM mode changes and AVX (for XCR0 syncing during worlds switching). */
1679	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1680
1681	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1682	Assert(!RT_HI_U32(uShadowCr4));
1683	pVmcb->guest.u64CR4 = uShadowCr4;
1684	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1685
1686	return VINF_SUCCESS;
1687	}
1688
1689
1690	/**
1691	* Exports the guest (or nested-guest) control registers into the VMCB.
1692	*
1693	* @returns VBox status code.
1694	* @param pVCpu The cross context virtual CPU structure.
1695	* @param pVmcb Pointer to the VM control block.
1696	*
1697	* @remarks No-long-jump zone!!!
1698	*/
1699	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1700	{
1701	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1702
1703	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1704	{
1705	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1706	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1707
1708	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1709	{
1710	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1711	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1712	}
1713
1714	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1715	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1716
1717	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1718	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1719	{
1720	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1721	if (RT_FAILURE(rc))
1722	return rc;
1723	}
1724
1725	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1726	}
1727	return VINF_SUCCESS;
1728	}
1729
1730
1731	/**
1732	* Exports the guest (or nested-guest) segment registers into the VMCB.
1733	*
1734	* @param pVCpu The cross context virtual CPU structure.
1735	* @param pVmcb Pointer to the VM control block.
1736	*
1737	* @remarks No-long-jump zone!!!
1738	*/
1739	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1740	{
1741	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1742	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1743
1744	/* Guest segment registers. */
1745	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1746	{
1747	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1748	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1749
1750	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1751	{
1752	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1753	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1754	}
1755
1756	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1757	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1758
1759	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1760	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1761
1762	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1763	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1764
1765	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1766	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1767
1768	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1769	}
1770
1771	/* Guest TR. */
1772	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1773	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1774
1775	/* Guest LDTR. */
1776	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1777	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1778
1779	/* Guest GDTR. */
1780	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1781	{
1782	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1783	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1784	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1785	}
1786
1787	/* Guest IDTR. */
1788	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1789	{
1790	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1791	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1792	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1793	}
1794
1795	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1796	\| HM_CHANGED_GUEST_TABLE_MASK);
1797	}
1798
1799
1800	/**
1801	* Exports the guest (or nested-guest) MSRs into the VMCB.
1802	*
1803	* @param pVCpu The cross context virtual CPU structure.
1804	* @param pVmcb Pointer to the VM control block.
1805	*
1806	* @remarks No-long-jump zone!!!
1807	*/
1808	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1809	{
1810	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1811	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1812
1813	/* Guest Sysenter MSRs. */
1814	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1815	{
1816	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1817	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1818
1819	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1820	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1821
1822	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1823	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1824	}
1825
1826	/*
1827	* Guest EFER MSR.
1828	* AMD-V requires guest EFER.SVME to be set. Weird.
1829	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1830	*/
1831	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1832	{
1833	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1834	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1835	}
1836
1837	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1838	if ( !CPUMIsGuestInLongModeEx(pCtx)
1839	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1840	{
1841	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1842	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1843	}
1844
1845	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1846	{
1847	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1848	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1849	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1850	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1851	}
1852
1853	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1854	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1855
1856	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1857	\| HM_CHANGED_GUEST_EFER_MSR
1858	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1859	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1860
1861	/*
1862	* Setup the PAT MSR (applicable for Nested Paging only).
1863	*
1864	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1865	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1866	*
1867	* While guests can modify and see the modified values through the shadow values,
1868	* we shall not honor any guest modifications of this MSR to ensure caching is always
1869	* enabled similar to how we clear CR0.CD and NW bits.
1870	*
1871	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1872	*/
1873	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1874
1875	/* Enable the last branch record bit if LBR virtualization is enabled. */
1876	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1877	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1878	}
1879
1880
1881	/**
1882	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1883	* the necessary intercepts accordingly.
1884	*
1885	* @param pVCpu The cross context virtual CPU structure.
1886	* @param pVmcb Pointer to the VM control block.
1887	*
1888	* @remarks No-long-jump zone!!!
1889	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1890	*/
1891	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1892	{
1893	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1894
1895	/** @todo Figure out stepping with nested-guest. */
1896	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1897	{
1898	/*
1899	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1900	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1901	* Instead, they are strictly only requested when the nested hypervisor intercepts
1902	* them -- handled while merging VMCB controls.
1903	*
1904	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1905	* then the nested-guest debug state should be actively loaded on the host so that
1906	* nested-guest reads/writes its own debug registers without causing VM-exits.
1907	*/
1908	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1909	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1910	&& !CPUMIsGuestDebugStateActive(pVCpu))
1911	{
1912	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1913	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1914	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1915	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1916	}
1917
1918	pVmcb->guest.u64DR6 = pCtx->dr[6];
1919	pVmcb->guest.u64DR7 = pCtx->dr[7];
1920	return;
1921	}
1922
1923	/*
1924	* Anyone single stepping on the host side? If so, we'll have to use the
1925	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1926	* the VMM level like the VT-x implementations does.
1927	*/
1928	bool fInterceptMovDRx = false;
1929	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1930	if (fStepping)
1931	{
1932	pVCpu->hmr0.s.fClearTrapFlag = true;
1933	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1934	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1935	}
1936
1937	if ( fStepping
1938	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1939	{
1940	/*
1941	* Use the combined guest and host DRx values found in the hypervisor
1942	* register set because the debugger has breakpoints active or someone
1943	* is single stepping on the host side.
1944	*
1945	* Note! DBGF expects a clean DR6 state before executing guest code.
1946	*/
1947	if (!CPUMIsHyperDebugStateActive(pVCpu))
1948	{
1949	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1950	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1951	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1952	}
1953
1954	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1955	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1956	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1957	{
1958	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1959	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1960	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1961	}
1962
1963	/** @todo If we cared, we could optimize to allow the guest to read registers
1964	* with the same values. */
1965	fInterceptMovDRx = true;
1966	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1967	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1968	}
1969	else
1970	{
1971	/*
1972	* Update DR6, DR7 with the guest values if necessary.
1973	*/
1974	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1975	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1976	{
1977	pVmcb->guest.u64DR7 = pCtx->dr[7];
1978	pVmcb->guest.u64DR6 = pCtx->dr[6];
1979	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1980	}
1981	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1982
1983	/*
1984	* If the guest has enabled debug registers, we need to load them prior to
1985	* executing guest code so they'll trigger at the right time.
1986	*/
1987	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1988	{
1989	if (!CPUMIsGuestDebugStateActive(pVCpu))
1990	{
1991	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1992	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1993	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1994	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1995	}
1996	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
1997	}
1998	/*
1999	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2000	* intercept #DB as DR6 is updated in the VMCB.
2001	*
2002	* Note! If we cared and dared, we could skip intercepting \#DB here.
2003	* However, \#DB shouldn't be performance critical, so we'll play safe
2004	* and keep the code similar to the VT-x code and always intercept it.
2005	*/
2006	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2007	fInterceptMovDRx = true;
2008	}
2009
2010	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2011	if (fInterceptMovDRx)
2012	{
2013	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2014	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2015	{
2016	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2017	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2018	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2019	}
2020	}
2021	else
2022	{
2023	if ( pVmcb->ctrl.u16InterceptRdDRx
2024	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2025	{
2026	pVmcb->ctrl.u16InterceptRdDRx = 0;
2027	pVmcb->ctrl.u16InterceptWrDRx = 0;
2028	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2029	}
2030	}
2031	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2032	}
2033
2034	/**
2035	* Exports the hardware virtualization state into the nested-guest
2036	* VMCB.
2037	*
2038	* @param pVCpu The cross context virtual CPU structure.
2039	* @param pVmcb Pointer to the VM control block.
2040	*
2041	* @remarks No-long-jump zone!!!
2042	*/
2043	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2044	{
2045	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2046
2047	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2048	{
2049	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2050	{
2051	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2052	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2053
2054	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2055	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2056	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2057	NOREF(pVM);
2058
2059	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2060	}
2061
2062	/*
2063	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2064	* since SVM doesn't have a preemption timer.
2065	*
2066	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2067	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2068	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2069	*/
2070	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2071	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2072	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2073	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2074	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2075	{
2076	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2077	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2078	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2079	}
2080	else
2081	{
2082	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2083	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2084	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2085	}
2086	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2087
2088	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2089	}
2090	}
2091
2092
2093	/**
2094	* Exports the guest APIC TPR state into the VMCB.
2095	*
2096	* @returns VBox status code.
2097	* @param pVCpu The cross context virtual CPU structure.
2098	* @param pVmcb Pointer to the VM control block.
2099	*/
2100	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2101	{
2102	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2103
2104	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2105	{
2106	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2107	if ( PDMHasApic(pVM)
2108	&& APICIsEnabled(pVCpu))
2109	{
2110	bool fPendingIntr;
2111	uint8_t u8Tpr;
2112	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2113	AssertRCReturn(rc, rc);
2114
2115	/* Assume that we need to trap all TPR accesses and thus need not check on
2116	every #VMEXIT if we should update the TPR. */
2117	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2118	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2119
2120	if (!pVM->hm.s.fTprPatchingActive)
2121	{
2122	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2123	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2124
2125	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2126	can deliver the interrupt to the guest. */
2127	if (fPendingIntr)
2128	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2129	else
2130	{
2131	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2132	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2133	}
2134
2135	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2136	}
2137	else
2138	{
2139	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2140	pVmcb->guest.u64LSTAR = u8Tpr;
2141	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2142
2143	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2144	if (fPendingIntr)
2145	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2146	else
2147	{
2148	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2149	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2150	}
2151	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2152	}
2153	}
2154	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2155	}
2156	return VINF_SUCCESS;
2157	}
2158
2159
2160	/**
2161	* Sets up the exception interrupts required for guest execution in the VMCB.
2162	*
2163	* @param pVCpu The cross context virtual CPU structure.
2164	* @param pVmcb Pointer to the VM control block.
2165	*
2166	* @remarks No-long-jump zone!!!
2167	*/
2168	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2169	{
2170	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2171
2172	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2173	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2174	{
2175	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2176	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2177	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2178	else
2179	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2180
2181	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2182	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledSwBreakpoints)
2183	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2184	else
2185	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2186
2187	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2188	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2189	}
2190	}
2191
2192
2193	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2194	/**
2195	* Merges guest and nested-guest intercepts for executing the nested-guest using
2196	* hardware-assisted SVM.
2197	*
2198	* This merges the guest and nested-guest intercepts in a way that if the outer
2199	* guest intercept is set we need to intercept it in the nested-guest as
2200	* well.
2201	*
2202	* @param pVCpu The cross context virtual CPU structure.
2203	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2204	*/
2205	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2206	{
2207	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2208	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2209	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2210	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2211
2212	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2213	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2214	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2215
2216	/* Always intercept CR4 writes for tracking PGM mode changes and AVX (for
2217	XCR0 syncing during worlds switching). */
2218	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2219
2220	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2221	if (!pVM->hmr0.s.fNestedPaging)
2222	{
2223	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2224	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2225	}
2226
2227	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2228	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2229	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2230
2231	/*
2232	* Merge the guest's exception intercepts into the nested-guest VMCB.
2233	*
2234	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2235	* while executing the nested-guest.
2236	*
2237	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2238	* be tweaked later depending on how we wish to implement breakpoints.
2239	*
2240	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2241	* loaded into their guests, not ours.
2242	*
2243	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2244	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2245	*/
2246	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2247	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2248	& ~( RT_BIT(X86_XCPT_UD)
2249	\| RT_BIT(X86_XCPT_BP)
2250	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2251	#else
2252	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2253	#endif
2254
2255	/*
2256	* Adjust intercepts while executing the nested-guest that differ from the
2257	* outer guest intercepts.
2258	*
2259	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2260	* that belong to the nested-guest to the outer guest.
2261	*
2262	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2263	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2264	*/
2265	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2266	\| SVM_CTRL_INTERCEPT_VMMCALL))
2267	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2268
2269	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2270	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2271
2272	/* Finally, update the VMCB clean bits. */
2273	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2274	}
2275	#endif
2276
2277
2278	/**
2279	* Enters the AMD-V session.
2280	*
2281	* @returns VBox status code.
2282	* @param pVCpu The cross context virtual CPU structure.
2283	*/
2284	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2285	{
2286	AssertPtr(pVCpu);
2287	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2288	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2289
2290	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2291	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2292	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2293
2294	pVCpu->hmr0.s.fLeaveDone = false;
2295	return VINF_SUCCESS;
2296	}
2297
2298
2299	/**
2300	* Thread-context callback for AMD-V.
2301	*
2302	* This is used together with RTThreadCtxHookCreate() on platforms which
2303	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2304	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2305	*
2306	* @param enmEvent The thread-context event.
2307	* @param pVCpu The cross context virtual CPU structure.
2308	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2309	* @thread EMT(pVCpu)
2310	*/
2311	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2312	{
2313	NOREF(fGlobalInit);
2314
2315	switch (enmEvent)
2316	{
2317	case RTTHREADCTXEVENT_OUT:
2318	{
2319	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2320	VMCPU_ASSERT_EMT(pVCpu);
2321
2322	/* No longjmps (log-flush, locks) in this fragile context. */
2323	VMMRZCallRing3Disable(pVCpu);
2324
2325	if (!pVCpu->hmr0.s.fLeaveDone)
2326	{
2327	hmR0SvmLeave(pVCpu, false /* fImportState */);
2328	pVCpu->hmr0.s.fLeaveDone = true;
2329	}
2330
2331	/* Leave HM context, takes care of local init (term). */
2332	int rc = HMR0LeaveCpu(pVCpu);
2333	AssertRC(rc); NOREF(rc);
2334
2335	/* Restore longjmp state. */
2336	VMMRZCallRing3Enable(pVCpu);
2337	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2338	break;
2339	}
2340
2341	case RTTHREADCTXEVENT_IN:
2342	{
2343	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2344	VMCPU_ASSERT_EMT(pVCpu);
2345
2346	/* No longjmps (log-flush, locks) in this fragile context. */
2347	VMMRZCallRing3Disable(pVCpu);
2348
2349	/*
2350	* Initialize the bare minimum state required for HM. This takes care of
2351	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2352	*/
2353	int rc = hmR0EnterCpu(pVCpu);
2354	AssertRC(rc); NOREF(rc);
2355	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2356	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2357
2358	pVCpu->hmr0.s.fLeaveDone = false;
2359
2360	/* Restore longjmp state. */
2361	VMMRZCallRing3Enable(pVCpu);
2362	break;
2363	}
2364
2365	default:
2366	break;
2367	}
2368	}
2369
2370
2371	/**
2372	* Saves the host state.
2373	*
2374	* @returns VBox status code.
2375	* @param pVCpu The cross context virtual CPU structure.
2376	*
2377	* @remarks No-long-jump zone!!!
2378	*/
2379	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2380	{
2381	NOREF(pVCpu);
2382
2383	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2384	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2385	return VINF_SUCCESS;
2386	}
2387
2388
2389	/**
2390	* Exports the guest or nested-guest state from the virtual-CPU context into the
2391	* VMCB.
2392	*
2393	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2394	* code based on the virtual-CPU mode.
2395	*
2396	* @returns VBox status code.
2397	* @param pVCpu The cross context virtual CPU structure.
2398	* @param pSvmTransient Pointer to the SVM-transient structure.
2399	*
2400	* @remarks No-long-jump zone!!!
2401	*/
2402	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2403	{
2404	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2405
2406	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2407	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2408	Assert(pVmcb);
2409
2410	pVmcb->guest.u64RIP = pCtx->rip;
2411	pVmcb->guest.u64RSP = pCtx->rsp;
2412	pVmcb->guest.u64RFlags = pCtx->eflags.u;
2413	pVmcb->guest.u64RAX = pCtx->rax;
2414
2415	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2416	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2417
2418	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2419	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2420	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2421	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2422	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2423
2424	ASMSetFlags(fEFlags);
2425
2426	if (!fIsNestedGuest)
2427	{
2428	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2429	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2430	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2431	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2432	}
2433
2434	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2435	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2436	\| HM_CHANGED_GUEST_RFLAGS
2437	\| HM_CHANGED_GUEST_GPRS_MASK
2438	\| HM_CHANGED_GUEST_X87
2439	\| HM_CHANGED_GUEST_SSE_AVX
2440	\| HM_CHANGED_GUEST_OTHER_XSAVE
2441	\| HM_CHANGED_GUEST_XCRx
2442	\| HM_CHANGED_GUEST_TSC_AUX
2443	\| HM_CHANGED_GUEST_OTHER_MSRS;
2444	if (fIsNestedGuest)
2445	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2446	\| HM_CHANGED_GUEST_APIC_TPR;
2447
2448	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2449	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2450
2451	#ifdef VBOX_STRICT
2452	/*
2453	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2454	* except for the host-context and/or shared host-guest context bits.
2455	*/
2456	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2457	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2458	("fCtxChanged=%#RX64\n", fCtxChanged));
2459
2460	/*
2461	* If we need to log state that isn't always imported, we'll need to import them here.
2462	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2463	*/
2464	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2465	#endif
2466
2467	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2468	return VINF_SUCCESS;
2469	}
2470
2471	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2472
2473	/**
2474	* Merges the guest and nested-guest MSR permission bitmap.
2475	*
2476	* If the guest is intercepting an MSR we need to intercept it regardless of
2477	* whether the nested-guest is intercepting it or not.
2478	*
2479	* @param pHostCpu The HM physical-CPU structure.
2480	* @param pVCpu The cross context virtual CPU structure.
2481	*
2482	* @remarks No-long-jmp zone!!!
2483	*/
2484	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2485	{
2486	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2487	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2488	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2489
2490	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2491	uint32_t const offRsvdQwords = 0x1800 >> 3;
2492	for (uint32_t i = 0; i < offRsvdQwords; i++)
2493	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2494	}
2495
2496
2497	/**
2498	* Caches the nested-guest VMCB fields before we modify them for execution using
2499	* hardware-assisted SVM.
2500	*
2501	* @returns true if the VMCB was previously already cached, false otherwise.
2502	* @param pVCpu The cross context virtual CPU structure.
2503	*
2504	* @sa HMNotifySvmNstGstVmexit.
2505	*/
2506	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2507	{
2508	/*
2509	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2510	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2511	*
2512	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2513	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2514	*/
2515	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2516	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2517	if (!fWasCached)
2518	{
2519	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2520	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2521	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2522	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2523	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2524	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2525	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2526	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2527	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2528	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2529	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2530	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2531	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2532	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2533	pVmcbNstGstCache->fCacheValid = true;
2534	Log4Func(("Cached VMCB fields\n"));
2535	}
2536
2537	return fWasCached;
2538	}
2539
2540
2541	/**
2542	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2543	*
2544	* This is done the first time we enter nested-guest execution using SVM R0
2545	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2546	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2547	*
2548	* @param pVCpu The cross context virtual CPU structure.
2549	*/
2550	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2551	{
2552	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2553	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2554
2555	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2556
2557	/*
2558	* First cache the nested-guest VMCB fields we may potentially modify.
2559	*/
2560	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2561	if (!fVmcbCached)
2562	{
2563	/*
2564	* The IOPM of the nested-guest can be ignored because the the guest always
2565	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2566	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2567	*/
2568	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2569
2570	/*
2571	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2572	* nested-paging suddenly while executing a VM (see assertion at the end of
2573	* Trap0eHandler() in PGMAllBth.h).
2574	*/
2575	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2576
2577	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2578	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2579
2580	/*
2581	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2582	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2583	*/
2584	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2585
2586	# ifdef DEBUG_ramshankar
2587	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2588	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2589	# endif
2590
2591	/*
2592	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2593	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2594	*/
2595	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2596	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2597	\| SVM_CTRL_INTERCEPT_VMLOAD;
2598
2599	/*
2600	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2601	* CLGI/STGI instructions executed by the nested-guest.
2602	*/
2603	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2604	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2605	\| SVM_CTRL_INTERCEPT_STGI;
2606
2607	/* Merge the guest and nested-guest intercepts. */
2608	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2609
2610	/* Update the VMCB clean bits. */
2611	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2612	}
2613	else
2614	{
2615	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2616	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2617	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2618	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2619	}
2620	}
2621
2622	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2623
2624	/**
2625	* Exports the state shared between the host and guest (or nested-guest) into
2626	* the VMCB.
2627	*
2628	* @param pVCpu The cross context virtual CPU structure.
2629	* @param pVmcb Pointer to the VM control block.
2630	*
2631	* @remarks No-long-jump zone!!!
2632	*/
2633	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2634	{
2635	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2636	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2637
2638	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2639	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2640
2641	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2642	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2643	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2644	}
2645
2646
2647	/**
2648	* Worker for SVMR0ImportStateOnDemand.
2649	*
2650	* @param pVCpu The cross context virtual CPU structure.
2651	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2652	*/
2653	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2654	{
2655	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2656
2657	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2658	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2659	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2660	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2661
2662	/*
2663	* We disable interrupts to make the updating of the state and in particular
2664	* the fExtrn modification atomic wrt to preemption hooks.
2665	*/
2666	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2667
2668	fWhat &= pCtx->fExtrn;
2669	if (fWhat)
2670	{
2671	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2672	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2673	{
2674	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2675	{
2676	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2677	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2678	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2679	}
2680	}
2681
2682	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2683	{
2684	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2685	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2686	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2687	}
2688	#endif
2689
2690	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2691	CPUMUpdateInterruptShadowEx(pCtx, pVmcbCtrl->IntShadow.n.u1IntShadow, pVmcbGuest->u64RIP);
2692
2693	if (fWhat & CPUMCTX_EXTRN_RIP)
2694	pCtx->rip = pVmcbGuest->u64RIP;
2695
2696	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2697	{
2698	pCtx->eflags.u = pVmcbGuest->u64RFlags;
2699	if (pVCpu->hmr0.s.fClearTrapFlag)
2700	{
2701	pVCpu->hmr0.s.fClearTrapFlag = false;
2702	pCtx->eflags.Bits.u1TF = 0;
2703	}
2704	}
2705
2706	if (fWhat & CPUMCTX_EXTRN_RSP)
2707	pCtx->rsp = pVmcbGuest->u64RSP;
2708
2709	if (fWhat & CPUMCTX_EXTRN_RAX)
2710	pCtx->rax = pVmcbGuest->u64RAX;
2711
2712	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2713	{
2714	if (fWhat & CPUMCTX_EXTRN_CS)
2715	{
2716	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2717	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2718	/** @todo SELM might need to be fixed as it too should not care about the
2719	* granularity bit. See @bugref{6785}. */
2720	if ( !pCtx->cs.Attr.n.u1Granularity
2721	&& pCtx->cs.Attr.n.u1Present
2722	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2723	{
2724	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2725	pCtx->cs.Attr.n.u1Granularity = 1;
2726	}
2727	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2728	}
2729	if (fWhat & CPUMCTX_EXTRN_SS)
2730	{
2731	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2732	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2733	/*
2734	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2735	* VMCB and uses that and thus it's possible that when the CPL changes during
2736	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2737	* AMD Fusion CPUs with 64-bit guests.
2738	*
2739	* See AMD spec. 15.5.1 "Basic operation".
2740	*/
2741	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2742	uint8_t const uCpl = pVmcbGuest->u8CPL;
2743	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2744	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2745	}
2746	if (fWhat & CPUMCTX_EXTRN_DS)
2747	{
2748	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2749	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2750	}
2751	if (fWhat & CPUMCTX_EXTRN_ES)
2752	{
2753	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2754	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2755	}
2756	if (fWhat & CPUMCTX_EXTRN_FS)
2757	{
2758	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2759	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2760	}
2761	if (fWhat & CPUMCTX_EXTRN_GS)
2762	{
2763	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2764	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2765	}
2766	}
2767
2768	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2769	{
2770	if (fWhat & CPUMCTX_EXTRN_TR)
2771	{
2772	/*
2773	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2774	* are used between Intel and AMD, see @bugref{6208#c39}.
2775	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2776	*/
2777	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2778	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2779	{
2780	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2781	\|\| CPUMIsGuestInLongModeEx(pCtx))
2782	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2783	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2784	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2785	}
2786	}
2787
2788	if (fWhat & CPUMCTX_EXTRN_LDTR)
2789	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2790
2791	if (fWhat & CPUMCTX_EXTRN_GDTR)
2792	{
2793	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2794	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2795	}
2796
2797	if (fWhat & CPUMCTX_EXTRN_IDTR)
2798	{
2799	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2800	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2801	}
2802	}
2803
2804	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2805	{
2806	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2807	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2808	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2809	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2810	}
2811
2812	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2813	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2814	{
2815	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2816	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2817	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2818	}
2819
2820	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2821	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2822
2823	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2824	{
2825	if (fWhat & CPUMCTX_EXTRN_DR6)
2826	{
2827	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2828	pCtx->dr[6] = pVmcbGuest->u64DR6;
2829	else
2830	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2831	}
2832
2833	if (fWhat & CPUMCTX_EXTRN_DR7)
2834	{
2835	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2836	pCtx->dr[7] = pVmcbGuest->u64DR7;
2837	else
2838	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2839	}
2840	}
2841
2842	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2843	{
2844	if (fWhat & CPUMCTX_EXTRN_CR0)
2845	{
2846	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2847	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2848	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2849	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2850	CPUMSetGuestCR0(pVCpu, uCr0);
2851	VMMRZCallRing3Enable(pVCpu);
2852	}
2853
2854	if (fWhat & CPUMCTX_EXTRN_CR2)
2855	pCtx->cr2 = pVmcbGuest->u64CR2;
2856
2857	if (fWhat & CPUMCTX_EXTRN_CR3)
2858	{
2859	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2860	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2861	{
2862	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2863	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2864	}
2865	}
2866
2867	/* Changes to CR4 are always intercepted. */
2868	}
2869
2870	/* Update fExtrn. */
2871	pCtx->fExtrn &= ~fWhat;
2872
2873	/* If everything has been imported, clear the HM keeper bit. */
2874	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2875	{
2876	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2877	Assert(!pCtx->fExtrn);
2878	}
2879	}
2880	else
2881	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2882
2883	ASMSetFlags(fEFlags);
2884
2885	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2886
2887	/*
2888	* Honor any pending CR3 updates.
2889	*
2890	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2891	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2892	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2893	* handling -> hmR0SvmImportGuestState() and here we are.
2894	*
2895	* The reason for such complicated handling is because VM-exits that call into PGM expect
2896	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2897	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2898	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2899	*
2900	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2901	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2902	*/
2903	if ( VMMRZCallRing3IsEnabled(pVCpu)
2904	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2905	{
2906	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2907	PGMUpdateCR3(pVCpu, pCtx->cr3);
2908	}
2909	}
2910
2911
2912	/**
2913	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2914	* context.
2915	*
2916	* Currently there is no residual state left in the CPU that is not updated in the
2917	* VMCB.
2918	*
2919	* @returns VBox status code.
2920	* @param pVCpu The cross context virtual CPU structure.
2921	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2922	*/
2923	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2924	{
2925	hmR0SvmImportGuestState(pVCpu, fWhat);
2926	return VINF_SUCCESS;
2927	}
2928
2929
2930	/**
2931	* Gets SVM \#VMEXIT auxiliary information.
2932	*
2933	* @returns VBox status code.
2934	* @param pVCpu The cross context virtual CPU structure.
2935	* @param pSvmExitAux Where to store the auxiliary info.
2936	*/
2937	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2938	{
2939	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2940	if (RT_LIKELY(pSvmTransient))
2941	{
2942	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2943	if (RT_LIKELY(pVmcb))
2944	{
2945	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2946	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2947	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2948	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2949	return VINF_SUCCESS;
2950	}
2951	return VERR_SVM_IPE_5;
2952	}
2953	return VERR_NOT_AVAILABLE;
2954	}
2955
2956
2957	/**
2958	* Does the necessary state syncing before returning to ring-3 for any reason
2959	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2960	*
2961	* @param pVCpu The cross context virtual CPU structure.
2962	* @param fImportState Whether to import the guest state from the VMCB back
2963	* to the guest-CPU context.
2964	*
2965	* @remarks No-long-jmp zone!!!
2966	*/
2967	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2968	{
2969	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2970	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2971
2972	/*
2973	* !!! IMPORTANT !!!
2974	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2975	*/
2976
2977	/* Save the guest state if necessary. */
2978	if (fImportState)
2979	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2980
2981	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2982	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2983	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2984
2985	/*
2986	* Restore host debug registers if necessary and resync on next R0 reentry.
2987	*/
2988	#ifdef VBOX_STRICT
2989	if (CPUMIsHyperDebugStateActive(pVCpu))
2990	{
2991	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2992	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2993	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2994	}
2995	#endif
2996	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2997	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2998	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2999
3000	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3001	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3002	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3003	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3004	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3005	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3006	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3007
3008	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3009	}
3010
3011
3012	/**
3013	* Leaves the AMD-V session.
3014	*
3015	* Only used while returning to ring-3 either due to longjump or exits to
3016	* ring-3.
3017	*
3018	* @returns VBox status code.
3019	* @param pVCpu The cross context virtual CPU structure.
3020	*/
3021	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3022	{
3023	HM_DISABLE_PREEMPT(pVCpu);
3024	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3025	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3026
3027	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3028	and done this from the SVMR0ThreadCtxCallback(). */
3029	if (!pVCpu->hmr0.s.fLeaveDone)
3030	{
3031	hmR0SvmLeave(pVCpu, true /* fImportState */);
3032	pVCpu->hmr0.s.fLeaveDone = true;
3033	}
3034
3035	/*
3036	* !!! IMPORTANT !!!
3037	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3038	*/
3039
3040	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3041	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3042	VMMR0ThreadCtxHookDisable(pVCpu);
3043
3044	/* Leave HM context. This takes care of local init (term). */
3045	int rc = HMR0LeaveCpu(pVCpu);
3046
3047	HM_RESTORE_PREEMPT();
3048	return rc;
3049	}
3050
3051
3052	/**
3053	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3054	* any remaining host state) before we go back to ring-3 due to an assertion.
3055	*
3056	* @param pVCpu The cross context virtual CPU structure.
3057	*/
3058	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3059	{
3060	/*
3061	* !!! IMPORTANT !!!
3062	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3063	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3064	*/
3065	VMMR0AssertionRemoveNotification(pVCpu);
3066	VMMRZCallRing3Disable(pVCpu);
3067	HM_DISABLE_PREEMPT(pVCpu);
3068
3069	/* Import the entire guest state. */
3070	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3071
3072	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3073	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3074
3075	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3076	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3077
3078	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3079	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3080	VMMR0ThreadCtxHookDisable(pVCpu);
3081
3082	/* Leave HM context. This takes care of local init (term). */
3083	HMR0LeaveCpu(pVCpu);
3084
3085	HM_RESTORE_PREEMPT();
3086	return VINF_SUCCESS;
3087	}
3088
3089
3090	/**
3091	* Take necessary actions before going back to ring-3.
3092	*
3093	* An action requires us to go back to ring-3. This function does the necessary
3094	* steps before we can safely return to ring-3. This is not the same as longjmps
3095	* to ring-3, this is voluntary.
3096	*
3097	* @returns Strict VBox status code.
3098	* @param pVCpu The cross context virtual CPU structure.
3099	* @param rcExit The reason for exiting to ring-3. Can be
3100	* VINF_VMM_UNKNOWN_RING3_CALL.
3101	*/
3102	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3103	{
3104	Assert(pVCpu);
3105	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3106
3107	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3108	VMMRZCallRing3Disable(pVCpu);
3109	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3110	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3111
3112	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3113	if (pVCpu->hm.s.Event.fPending)
3114	{
3115	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3116	Assert(!pVCpu->hm.s.Event.fPending);
3117	}
3118
3119	/* Sync. the necessary state for going back to ring-3. */
3120	hmR0SvmLeaveSession(pVCpu);
3121	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3122
3123	/* Thread-context hooks are unregistered at this point!!! */
3124	/* Ring-3 callback notifications are unregistered at this point!!! */
3125
3126	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3127	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3128	\| CPUM_CHANGED_LDTR
3129	\| CPUM_CHANGED_GDTR
3130	\| CPUM_CHANGED_IDTR
3131	\| CPUM_CHANGED_TR
3132	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3133	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3134	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3135	{
3136	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3137	}
3138
3139	/* Update the exit-to-ring 3 reason. */
3140	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3141
3142	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3143	if ( rcExit != VINF_EM_RAW_INTERRUPT
3144	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3145	{
3146	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3147	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3148	}
3149
3150	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3151	VMMRZCallRing3Enable(pVCpu);
3152
3153	/*
3154	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3155	* and if we're injecting an event we should have a TRPM trap pending.
3156	*/
3157	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3158	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3159	VERR_SVM_IPE_5);
3160	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3161	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3162	VERR_SVM_IPE_4);
3163
3164	return rcExit;
3165	}
3166
3167
3168	/**
3169	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3170	* intercepts.
3171	*
3172	* @param pVCpu The cross context virtual CPU structure.
3173	* @param pVmcb Pointer to the VM control block.
3174	*
3175	* @remarks No-long-jump zone!!!
3176	*/
3177	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3178	{
3179	/*
3180	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3181	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3182	* RDTSC/RDTSCP as well.
3183	*/
3184	bool fParavirtTsc;
3185	uint64_t uTscOffset;
3186	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3187
3188	bool fIntercept;
3189	if (fCanUseRealTsc)
3190	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3191	else
3192	{
3193	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3194	fIntercept = true;
3195	}
3196
3197	if (!fIntercept)
3198	{
3199	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3200	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3201	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3202	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3203	#endif
3204
3205	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3206	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3207	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3208	}
3209
3210	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3211	information before every VM-entry, hence we have nothing to do here at the moment. */
3212	if (fParavirtTsc)
3213	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3214	}
3215
3216
3217	/**
3218	* Sets an event as a pending event to be injected into the guest.
3219	*
3220	* @param pVCpu The cross context virtual CPU structure.
3221	* @param pEvent Pointer to the SVM event.
3222	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3223	* page-fault.
3224	*
3225	* @remarks Statistics counter assumes this is a guest event being reflected to
3226	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3227	*/
3228	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3229	{
3230	Assert(!pVCpu->hm.s.Event.fPending);
3231	Assert(pEvent->n.u1Valid);
3232
3233	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3234	pVCpu->hm.s.Event.fPending = true;
3235	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3236
3237	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3238	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3239	}
3240
3241
3242	/**
3243	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3244	*
3245	* @param pVCpu The cross context virtual CPU structure.
3246	*/
3247	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3248	{
3249	SVMEVENT Event;
3250	Event.u = 0;
3251	Event.n.u1Valid = 1;
3252	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3253	Event.n.u8Vector = X86_XCPT_DE;
3254	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3255	}
3256
3257
3258	/**
3259	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3260	*
3261	* @param pVCpu The cross context virtual CPU structure.
3262	*/
3263	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3264	{
3265	SVMEVENT Event;
3266	Event.u = 0;
3267	Event.n.u1Valid = 1;
3268	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3269	Event.n.u8Vector = X86_XCPT_UD;
3270	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3271	}
3272
3273
3274	/**
3275	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3276	*
3277	* @param pVCpu The cross context virtual CPU structure.
3278	*/
3279	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3280	{
3281	SVMEVENT Event;
3282	Event.u = 0;
3283	Event.n.u1Valid = 1;
3284	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3285	Event.n.u8Vector = X86_XCPT_DB;
3286	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3287	}
3288
3289
3290	/**
3291	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3292	*
3293	* @param pVCpu The cross context virtual CPU structure.
3294	* @param u32ErrCode The error-code for the page-fault.
3295	* @param uFaultAddress The page fault address (CR2).
3296	*
3297	* @remarks This updates the guest CR2 with @a uFaultAddress!
3298	*/
3299	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3300	{
3301	SVMEVENT Event;
3302	Event.u = 0;
3303	Event.n.u1Valid = 1;
3304	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3305	Event.n.u8Vector = X86_XCPT_PF;
3306	Event.n.u1ErrorCodeValid = 1;
3307	Event.n.u32ErrorCode = u32ErrCode;
3308
3309	/* Update CR2 of the guest. */
3310	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3311	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3312	{
3313	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3314	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3315	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3316	}
3317
3318	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3319	}
3320
3321
3322	/**
3323	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3324	*
3325	* @param pVCpu The cross context virtual CPU structure.
3326	*/
3327	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3328	{
3329	SVMEVENT Event;
3330	Event.u = 0;
3331	Event.n.u1Valid = 1;
3332	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3333	Event.n.u8Vector = X86_XCPT_MF;
3334	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3335	}
3336
3337
3338	/**
3339	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3340	*
3341	* @param pVCpu The cross context virtual CPU structure.
3342	*/
3343	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3344	{
3345	SVMEVENT Event;
3346	Event.u = 0;
3347	Event.n.u1Valid = 1;
3348	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3349	Event.n.u8Vector = X86_XCPT_DF;
3350	Event.n.u1ErrorCodeValid = 1;
3351	Event.n.u32ErrorCode = 0;
3352	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3353	}
3354
3355
3356	/**
3357	* Injects an event into the guest upon VMRUN by updating the relevant field
3358	* in the VMCB.
3359	*
3360	* @param pVCpu The cross context virtual CPU structure.
3361	* @param pVmcb Pointer to the guest VM control block.
3362	* @param pEvent Pointer to the event.
3363	*
3364	* @remarks No-long-jump zone!!!
3365	* @remarks Requires CR0!
3366	*/
3367	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3368	{
3369	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3370	pVmcb->ctrl.EventInject.u = pEvent->u;
3371	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3372	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3373	{
3374	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3375	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3376	}
3377	else
3378	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3379	RT_NOREF(pVCpu);
3380
3381	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3382	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3383	}
3384
3385
3386
3387	/**
3388	* Converts any TRPM trap into a pending HM event. This is typically used when
3389	* entering from ring-3 (not longjmp returns).
3390	*
3391	* @param pVCpu The cross context virtual CPU structure.
3392	*/
3393	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3394	{
3395	Assert(TRPMHasTrap(pVCpu));
3396	Assert(!pVCpu->hm.s.Event.fPending);
3397
3398	uint8_t uVector;
3399	TRPMEVENT enmTrpmEvent;
3400	uint32_t uErrCode;
3401	RTGCUINTPTR GCPtrFaultAddress;
3402	uint8_t cbInstr;
3403
3404	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3405	AssertRC(rc);
3406
3407	SVMEVENT Event;
3408	Event.u = 0;
3409	Event.n.u1Valid = 1;
3410	Event.n.u8Vector = uVector;
3411
3412	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3413	if (enmTrpmEvent == TRPM_TRAP)
3414	{
3415	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3416	switch (uVector)
3417	{
3418	case X86_XCPT_BP:
3419	case X86_XCPT_OF:
3420	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3421	RT_FALL_THRU();
3422
3423	case X86_XCPT_PF:
3424	case X86_XCPT_DF:
3425	case X86_XCPT_TS:
3426	case X86_XCPT_NP:
3427	case X86_XCPT_SS:
3428	case X86_XCPT_GP:
3429	case X86_XCPT_AC:
3430	{
3431	Event.n.u1ErrorCodeValid = 1;
3432	Event.n.u32ErrorCode = uErrCode;
3433	break;
3434	}
3435	}
3436	}
3437	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3438	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3439	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3440	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3441	else if (enmTrpmEvent == TRPM_NMI)
3442	Event.n.u3Type = SVM_EVENT_NMI;
3443	else
3444	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3445
3446	rc = TRPMResetTrap(pVCpu);
3447	AssertRC(rc);
3448
3449	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3450	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3451
3452	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3453	}
3454
3455
3456	/**
3457	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3458	* AMD-V to execute any instruction.
3459	*
3460	* @param pVCpu The cross context virtual CPU structure.
3461	*/
3462	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3463	{
3464	Assert(pVCpu->hm.s.Event.fPending);
3465	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3466
3467	SVMEVENT Event;
3468	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3469
3470	uint8_t uVector = Event.n.u8Vector;
3471	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3472
3473	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3474
3475	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3476	AssertRC(rc);
3477
3478	if (Event.n.u1ErrorCodeValid)
3479	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3480
3481	if ( enmTrapType == TRPM_TRAP
3482	&& uVector == X86_XCPT_PF)
3483	{
3484	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3485	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3486	}
3487	else if (enmTrapType == TRPM_SOFTWARE_INT)
3488	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3489	pVCpu->hm.s.Event.fPending = false;
3490	}
3491
3492
3493	/**
3494	* Sets the virtual interrupt intercept control in the VMCB.
3495	*
3496	* @param pVCpu The cross context virtual CPU structure.
3497	* @param pVmcb Pointer to the VM control block.
3498	*/
3499	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3500	{
3501	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3502
3503	/*
3504	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3505	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3506	* (updating ISR at the right time) and inject the interrupt.
3507	*
3508	* With AVIC is supported, we could make use of the asynchronously delivery without
3509	* #VMEXIT and we would be passing the AVIC page to SVM.
3510	*
3511	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3512	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3513	*/
3514	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3515	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3516	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3517	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3518	Log4(("Set VINTR intercept\n"));
3519	}
3520
3521
3522	/**
3523	* Clears the virtual interrupt intercept control in the VMCB as
3524	* we are figured the guest is unable process any interrupts
3525	* at this point of time.
3526	*
3527	* @param pVCpu The cross context virtual CPU structure.
3528	* @param pVmcb Pointer to the VM control block.
3529	*/
3530	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3531	{
3532	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3533
3534	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3535	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3536	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3537	{
3538	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3539	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3540	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3541	Log4(("Cleared VINTR intercept\n"));
3542	}
3543	}
3544
3545
3546	/**
3547	* Evaluates the event to be delivered to the guest and sets it as the pending
3548	* event.
3549	*
3550	* @returns Strict VBox status code.
3551	* @param pVCpu The cross context virtual CPU structure.
3552	* @param pSvmTransient Pointer to the SVM transient structure.
3553	*/
3554	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3555	{
3556	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3557	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3558	\| CPUMCTX_EXTRN_RFLAGS
3559	\| CPUMCTX_EXTRN_INHIBIT_INT
3560	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3561
3562	Assert(!pVCpu->hm.s.Event.fPending);
3563	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3564	Assert(pVmcb);
3565
3566	bool const fGif = CPUMGetGuestGif(pCtx);
3567	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(pCtx);
3568	bool const fBlockNmi = CPUMAreInterruptsInhibitedByNmi(pCtx);
3569
3570	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3571	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3572	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3573
3574	/** @todo SMI. SMIs take priority over NMIs. */
3575
3576	/*
3577	* Check if the guest or nested-guest can receive NMIs.
3578	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3579	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3580	*/
3581	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3582	&& !fBlockNmi)
3583	{
3584	if ( fGif
3585	&& !fIntShadow)
3586	{
3587	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3588	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3589	{
3590	Log4(("Intercepting NMI -> #VMEXIT\n"));
3591	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3592	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3593	}
3594	#endif
3595	Log4(("Setting NMI pending for injection\n"));
3596	SVMEVENT Event;
3597	Event.u = 0;
3598	Event.n.u1Valid = 1;
3599	Event.n.u8Vector = X86_XCPT_NMI;
3600	Event.n.u3Type = SVM_EVENT_NMI;
3601	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3602	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3603	}
3604	else if (!fGif)
3605	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3606	else if (!pSvmTransient->fIsNestedGuest)
3607	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3608	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3609	}
3610	/*
3611	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3612	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3613	* it from the APIC device.
3614	*
3615	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3616	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3617	* do that work when we execute nested-guest code esp. since all the required information
3618	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3619	* the virtual interrupt controller.
3620	*
3621	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3622	*/
3623	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3624	&& !pVCpu->hm.s.fSingleInstruction)
3625	{
3626	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u & X86_EFL_IF)
3627	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3628	if ( fGif
3629	&& !fBlockInt
3630	&& !fIntShadow)
3631	{
3632	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3633	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3634	{
3635	Log4(("Intercepting INTR -> #VMEXIT\n"));
3636	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3637	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3638	}
3639	#endif
3640	uint8_t u8Interrupt;
3641	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3642	if (RT_SUCCESS(rc))
3643	{
3644	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3645	SVMEVENT Event;
3646	Event.u = 0;
3647	Event.n.u1Valid = 1;
3648	Event.n.u8Vector = u8Interrupt;
3649	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3650	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3651	}
3652	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3653	{
3654	/*
3655	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3656	* updated eventually when the TPR is written by the guest.
3657	*/
3658	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3659	}
3660	else
3661	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3662	}
3663	else if (!fGif)
3664	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3665	else if (!pSvmTransient->fIsNestedGuest)
3666	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3667	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3668	}
3669
3670	return VINF_SUCCESS;
3671	}
3672
3673
3674	/**
3675	* Injects any pending events into the guest (or nested-guest).
3676	*
3677	* @param pVCpu The cross context virtual CPU structure.
3678	* @param pVmcb Pointer to the VM control block.
3679	*
3680	* @remarks Must only be called when we are guaranteed to enter
3681	* hardware-assisted SVM execution and not return to ring-3
3682	* prematurely.
3683	*/
3684	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3685	{
3686	Assert(!TRPMHasTrap(pVCpu));
3687	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3688
3689	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx);
3690	#ifdef VBOX_STRICT
3691	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3692	bool const fGif = CPUMGetGuestGif(pCtx);
3693	bool fAllowInt = fGif;
3694	if (fGif)
3695	{
3696	/*
3697	* For nested-guests we have no way to determine if we're injecting a physical or
3698	* virtual interrupt at this point. Hence the partial verification below.
3699	*/
3700	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3701	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3702	else
3703	fAllowInt = RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
3704	}
3705	#endif
3706
3707	if (pVCpu->hm.s.Event.fPending)
3708	{
3709	SVMEVENT Event;
3710	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3711	Assert(Event.n.u1Valid);
3712
3713	/*
3714	* Validate event injection pre-conditions.
3715	*/
3716	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3717	{
3718	Assert(fAllowInt);
3719	Assert(!fIntShadow);
3720	}
3721	else if (Event.n.u3Type == SVM_EVENT_NMI)
3722	{
3723	Assert(fGif);
3724	Assert(!fIntShadow);
3725	}
3726
3727	/*
3728	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3729	* do this only when we are surely going to inject the NMI as otherwise if we return
3730	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3731	* SVM R0.
3732	*
3733	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3734	* which will set the VMCS field after actually delivering the NMI which we read on
3735	* VM-exit to determine the state.
3736	*/
3737	if ( Event.n.u3Type == SVM_EVENT_NMI
3738	&& Event.n.u8Vector == X86_XCPT_NMI)
3739	CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3740
3741	/*
3742	* Inject it (update VMCB for injection by the hardware).
3743	*/
3744	Log4(("Injecting pending HM event\n"));
3745	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3746	pVCpu->hm.s.Event.fPending = false;
3747
3748	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3749	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3750	else
3751	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3752	}
3753	else
3754	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3755
3756	/*
3757	* We could have injected an NMI through IEM and continue guest execution using
3758	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3759	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3760	*/
3761	if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
3762	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3763
3764	/*
3765	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3766	*
3767	* For nested-guests: We need to update it too for the scenario where IEM executes
3768	* the nested-guest but execution later continues here with an interrupt shadow active.
3769	*/
3770	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3771	}
3772
3773
3774	/**
3775	* Reports world-switch error and dumps some useful debug info.
3776	*
3777	* @param pVCpu The cross context virtual CPU structure.
3778	* @param rcVMRun The return code from VMRUN (or
3779	* VERR_SVM_INVALID_GUEST_STATE for invalid
3780	* guest-state).
3781	*/
3782	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3783	{
3784	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3785	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3786	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3787
3788	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3789	{
3790	#ifdef VBOX_STRICT
3791	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3792	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3793	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3794	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3795	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3796	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3797	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3798	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3799	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3800	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3801	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3802	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3803
3804	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3805	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3806	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3807
3808	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3809	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3810	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3811	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3812	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3813	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3814	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3815	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3816	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3817	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3818	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3819	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3820
3821	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3822	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3823	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3824	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3825	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3826	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3827	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3828	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3829	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3830	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3831	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3832	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3833	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3834	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3835	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3836	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3837	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3838	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3839	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3840	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3841
3842	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3843
3844	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3845	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3846
3847	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3848	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3849	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3850	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3851	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3852	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3853	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3854	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3855	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3856	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3857	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3858	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3859	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3860	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3861	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3862	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3863	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3864	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3865	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3866	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3867
3868	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3869	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3870
3871	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3872	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3873	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3874	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3875
3876	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3877	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3878
3879	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3880	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3881	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3882	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3883
3884	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3885	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3886	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3887	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3888	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3889	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3890	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3891
3892	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3893	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3894	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3895	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3896
3897	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3898	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3899	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3900
3901	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3902	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3903	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3904	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3905	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3906	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3907	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3908	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3909	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3910	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3911	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3912	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3913
3914	NOREF(pVmcb);
3915	#endif /* VBOX_STRICT */
3916	}
3917	else
3918	Log4Func(("rcVMRun=%d\n", rcVMRun));
3919	}
3920
3921
3922	/**
3923	* Check per-VM and per-VCPU force flag actions that require us to go back to
3924	* ring-3 for one reason or another.
3925	*
3926	* @returns Strict VBox status code (information status code included).
3927	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3928	* ring-3.
3929	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3930	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3931	* interrupts)
3932	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3933	* all EMTs to be in ring-3.
3934	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3935	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3936	* to the EM loop.
3937	*
3938	* @param pVCpu The cross context virtual CPU structure.
3939	*/
3940	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3941	{
3942	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3943
3944	/* Could happen as a result of longjump. */
3945	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3946	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3947
3948	/* Update pending interrupts into the APIC's IRR. */
3949	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3950	APICUpdatePendingInterrupts(pVCpu);
3951
3952	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3953	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3954	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3955	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3956	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3957	{
3958	/* Pending PGM C3 sync. */
3959	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3960	{
3961	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3962	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3963	if (rc != VINF_SUCCESS)
3964	{
3965	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3966	return rc;
3967	}
3968	}
3969
3970	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3971	/* -XXX- what was that about single stepping? */
3972	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3973	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3974	{
3975	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3976	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3977	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3978	return rc;
3979	}
3980
3981	/* Pending VM request packets, such as hardware interrupts. */
3982	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3983	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3984	{
3985	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
3986	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3987	return VINF_EM_PENDING_REQUEST;
3988	}
3989
3990	/* Pending PGM pool flushes. */
3991	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3992	{
3993	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
3994	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
3995	return VINF_PGM_POOL_FLUSH_PENDING;
3996	}
3997
3998	/* Pending DMA requests. */
3999	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4000	{
4001	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4002	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4003	return VINF_EM_RAW_TO_R3;
4004	}
4005	}
4006
4007	return VINF_SUCCESS;
4008	}
4009
4010
4011	/**
4012	* Does the preparations before executing guest code in AMD-V.
4013	*
4014	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4015	* recompiler. We must be cautious what we do here regarding committing
4016	* guest-state information into the VMCB assuming we assuredly execute the guest
4017	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4018	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4019	* that the recompiler can (and should) use them when it resumes guest
4020	* execution. Otherwise such operations must be done when we can no longer
4021	* exit to ring-3.
4022	*
4023	* @returns Strict VBox status code (informational status codes included).
4024	* @retval VINF_SUCCESS if we can proceed with running the guest.
4025	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4026	*
4027	* @param pVCpu The cross context virtual CPU structure.
4028	* @param pSvmTransient Pointer to the SVM transient structure.
4029	*/
4030	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4031	{
4032	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4033
4034	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4035	if (pSvmTransient->fIsNestedGuest)
4036	{
4037	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4038	return VINF_EM_RESCHEDULE_REM;
4039	}
4040	#endif
4041
4042	/* Check force flag actions that might require us to go back to ring-3. */
4043	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4044	if (rc != VINF_SUCCESS)
4045	return rc;
4046
4047	if (TRPMHasTrap(pVCpu))
4048	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4049	else if (!pVCpu->hm.s.Event.fPending)
4050	{
4051	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4052	if ( rc != VINF_SUCCESS
4053	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4054	{
4055	/* If a nested-guest VM-exit occurred, bail. */
4056	if (pSvmTransient->fIsNestedGuest)
4057	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4058	return rc;
4059	}
4060	}
4061
4062	/*
4063	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4064	* Just do it in software, see @bugref{8411}.
4065	* NB: If we could continue a task switch exit we wouldn't need to do this.
4066	*/
4067	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4068	if (RT_UNLIKELY( !g_fHmSvmFeatures
4069	&& pVCpu->hm.s.Event.fPending
4070	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4071	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4072
4073	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4074	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4075	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4076	#endif
4077
4078	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4079	/*
4080	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4081	*/
4082	if (pSvmTransient->fIsNestedGuest)
4083	hmR0SvmSetupVmcbNested(pVCpu);
4084	#endif
4085
4086	/*
4087	* Export the guest state bits that are not shared with the host in any way as we can
4088	* longjmp or get preempted in the midst of exporting some of the state.
4089	*/
4090	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4091	AssertRCReturn(rc, rc);
4092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4093
4094	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4095	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4096
4097	/*
4098	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4099	* world-switch so we can update it on the way back if the guest changed the TPR.
4100	*/
4101	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4102	{
4103	Assert(!pSvmTransient->fIsNestedGuest);
4104	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4105	if (pVM->hm.s.fTprPatchingActive)
4106	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4107	else
4108	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4109	}
4110
4111	/*
4112	* No longjmps to ring-3 from this point on!!!
4113	*
4114	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4115	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4116	*/
4117	VMMRZCallRing3Disable(pVCpu);
4118
4119	/*
4120	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4121	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4122	* preemption disabled for a while. Since this is purly to aid the
4123	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4124	* disable interrupt on NT.
4125	*
4126	* We need to check for force-flags that could've possible been altered since we last
4127	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4128	* see @bugref{6398}).
4129	*
4130	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4131	* to ring-3 before executing guest code.
4132	*/
4133	pSvmTransient->fEFlags = ASMIntDisableFlags();
4134	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4135	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4136	{
4137	ASMSetFlags(pSvmTransient->fEFlags);
4138	VMMRZCallRing3Enable(pVCpu);
4139	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4140	return VINF_EM_RAW_TO_R3;
4141	}
4142	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4143	{
4144	ASMSetFlags(pSvmTransient->fEFlags);
4145	VMMRZCallRing3Enable(pVCpu);
4146	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4147	return VINF_EM_RAW_INTERRUPT;
4148	}
4149
4150	return VINF_SUCCESS;
4151	}
4152
4153
4154	/**
4155	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4156	* doing so.
4157	*
4158	* This means there is no backing out to ring-3 or anywhere else at this point.
4159	*
4160	* @param pVCpu The cross context virtual CPU structure.
4161	* @param pSvmTransient Pointer to the SVM transient structure.
4162	*
4163	* @remarks Called with preemption disabled.
4164	* @remarks No-long-jump zone!!!
4165	*/
4166	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4167	{
4168	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4169	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4170
4171	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4172	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4173
4174	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4175	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4176
4177	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4178
4179	if (!CPUMIsGuestFPUStateActive(pVCpu))
4180	{
4181	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4182	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4183	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4184	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4185	}
4186
4187	/* Load the state shared between host and guest (FPU, debug). */
4188	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4189	hmR0SvmExportSharedState(pVCpu, pVmcb);
4190
4191	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4192	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4193
4194	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4195	RTCPUID const idHostCpu = pHostCpu->idCpu;
4196	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4197
4198	/* Setup TSC offsetting. */
4199	if ( pSvmTransient->fUpdateTscOffsetting
4200	\|\| fMigratedHostCpu)
4201	{
4202	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4203	pSvmTransient->fUpdateTscOffsetting = false;
4204	}
4205
4206	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4207	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4208	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4209	else
4210	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4211
4212	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4213	if (fMigratedHostCpu)
4214	pVmcb->ctrl.u32VmcbCleanBits = 0;
4215
4216	/* Store status of the shared guest-host state at the time of VMRUN. */
4217	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4218	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4219
4220	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4221	uint8_t *pbMsrBitmap;
4222	if (!pSvmTransient->fIsNestedGuest)
4223	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4224	else
4225	{
4226	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4227	* MSRPM and only perform this if it changed also use EVEX.POR when it
4228	* does. */
4229	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4230
4231	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4232	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4233	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4234	}
4235	#else
4236	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4237	#endif
4238
4239	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4240	/* Flush the appropriate tagged-TLB entries. */
4241	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4242	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4243
4244	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4245
4246	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4247	to start executing. */
4248
4249	/*
4250	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4251	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4252	*
4253	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4254	*/
4255	if ( g_CpumHostFeatures.s.fRdTscP
4256	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4257	{
4258	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4259	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4260	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4261	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4262	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4263	pSvmTransient->fRestoreTscAuxMsr = true;
4264	}
4265	else
4266	{
4267	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4268	pSvmTransient->fRestoreTscAuxMsr = false;
4269	}
4270	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4271
4272	/*
4273	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4274	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4275	* from the VMCB.
4276	*/
4277	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4278	if (!fSupportsVmcbCleanBits)
4279	pVmcb->ctrl.u32VmcbCleanBits = 0;
4280	}
4281
4282
4283	/**
4284	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4285	*
4286	* @returns VBox strict status code.
4287	* @param pVCpu The cross context virtual CPU structure.
4288	* @param HCPhysVmcb The host physical address of the VMCB.
4289	*
4290	* @remarks No-long-jump zone!!!
4291	*/
4292	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4293	{
4294	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4295	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4296	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4297	}
4298
4299
4300	/**
4301	* Performs some essential restoration of state after running guest (or
4302	* nested-guest) code in AMD-V.
4303	*
4304	* @param pVCpu The cross context virtual CPU structure.
4305	* @param pSvmTransient Pointer to the SVM transient structure.
4306	* @param rcVMRun Return code of VMRUN.
4307	*
4308	* @remarks Called with interrupts disabled.
4309	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4310	* unconditionally when it is safe to do so.
4311	*/
4312	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4313	{
4314	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4315
4316	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4317	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4318
4319	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4320	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4321
4322	/* TSC read must be done early for maximum accuracy. */
4323	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4324	{
4325	if (!pSvmTransient->fIsNestedGuest)
4326	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4327	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4328	else
4329	{
4330	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4331	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4332	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4333	}
4334	#endif
4335	}
4336
4337	if (pSvmTransient->fRestoreTscAuxMsr)
4338	{
4339	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4340	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4341	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4342	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4343	}
4344
4345	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4346	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4347	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4348	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4349
4350	Assert(!(ASMGetFlags() & X86_EFL_IF));
4351	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4352	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4353
4354	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4355	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4356	{
4357	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4358	return;
4359	}
4360
4361	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4362	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4363	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4364	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4365
4366	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4367	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4368	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4369	#else
4370	/*
4371	* Always import the following:
4372	*
4373	* - RIP for exit optimizations and evaluating event injection on re-entry.
4374	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4375	* state on preemption.
4376	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4377	* - CS for exit optimizations.
4378	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4379	* assembly switcher code.
4380	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4381	*/
4382	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4383	\| CPUMCTX_EXTRN_RFLAGS
4384	\| CPUMCTX_EXTRN_RAX
4385	\| CPUMCTX_EXTRN_RSP
4386	\| CPUMCTX_EXTRN_CS
4387	\| CPUMCTX_EXTRN_HWVIRT
4388	\| CPUMCTX_EXTRN_INHIBIT_INT
4389	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4390	\| HMSVM_CPUMCTX_SHARED_STATE);
4391	#endif
4392
4393	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4394	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4395	{
4396	Assert(!pSvmTransient->fIsNestedGuest);
4397	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4398	if ( pVM->hm.s.fTprPatchingActive
4399	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4400	{
4401	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4402	AssertRC(rc);
4403	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4404	}
4405	/* Sync TPR when we aren't intercepting CR8 writes. */
4406	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4407	{
4408	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4409	AssertRC(rc);
4410	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4411	}
4412	}
4413
4414	#ifdef DEBUG_ramshankar
4415	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4416	{
4417	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4418	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4419	0 /* uVerbose */);
4420	}
4421	#endif
4422
4423	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4424	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4425	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4426	}
4427
4428
4429	/**
4430	* Runs the guest code using AMD-V.
4431	*
4432	* @returns Strict VBox status code.
4433	* @param pVCpu The cross context virtual CPU structure.
4434	* @param pcLoops Pointer to the number of executed loops.
4435	*/
4436	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4437	{
4438	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4439	Assert(pcLoops);
4440	Assert(*pcLoops <= cMaxResumeLoops);
4441
4442	SVMTRANSIENT SvmTransient;
4443	RT_ZERO(SvmTransient);
4444	SvmTransient.fUpdateTscOffsetting = true;
4445	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4446
4447	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4448	for (;;)
4449	{
4450	Assert(!HMR0SuspendPending());
4451	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4452
4453	/* Preparatory work for running nested-guest code, this may force us to return to
4454	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4455	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4456	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4457	if (rc != VINF_SUCCESS)
4458	break;
4459
4460	/*
4461	* No longjmps to ring-3 from this point on!!!
4462	*
4463	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4464	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4465	*/
4466	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4467	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4468
4469	/* Restore any residual host-state and save any bits shared between host and guest
4470	into the guest-CPU state. Re-enables interrupts! */
4471	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4472
4473	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4474	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4475	{
4476	if (rc == VINF_SUCCESS)
4477	rc = VERR_SVM_INVALID_GUEST_STATE;
4478	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4479	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4480	break;
4481	}
4482
4483	/* Handle the #VMEXIT. */
4484	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4485	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4486	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4487	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4488	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4489	if (rc != VINF_SUCCESS)
4490	break;
4491	if (++(*pcLoops) >= cMaxResumeLoops)
4492	{
4493	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4494	rc = VINF_EM_RAW_INTERRUPT;
4495	break;
4496	}
4497	}
4498
4499	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4500	return rc;
4501	}
4502
4503
4504	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4505	/**
4506	* Runs the nested-guest code using AMD-V.
4507	*
4508	* @returns Strict VBox status code.
4509	* @param pVCpu The cross context virtual CPU structure.
4510	* @param pcLoops Pointer to the number of executed loops. If we're switching
4511	* from the guest-code execution loop to this nested-guest
4512	* execution loop pass the remainder value, else pass 0.
4513	*/
4514	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4515	{
4516	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4517	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4518	Assert(pcLoops);
4519	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4520	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4521	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4522
4523	SVMTRANSIENT SvmTransient;
4524	RT_ZERO(SvmTransient);
4525	SvmTransient.fUpdateTscOffsetting = true;
4526	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4527	SvmTransient.fIsNestedGuest = true;
4528
4529	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4530	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4531
4532	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4533	for (;;)
4534	{
4535	Assert(!HMR0SuspendPending());
4536	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4537
4538	/* Preparatory work for running nested-guest code, this may force us to return to
4539	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4540	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4541	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4542	if ( rc != VINF_SUCCESS
4543	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4544	break;
4545
4546	/*
4547	* No longjmps to ring-3 from this point on!!!
4548	*
4549	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4550	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4551	*/
4552	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4553
4554	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4555
4556	/* Restore any residual host-state and save any bits shared between host and guest
4557	into the guest-CPU state. Re-enables interrupts! */
4558	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4559
4560	if (RT_LIKELY( rc == VINF_SUCCESS
4561	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4562	{ /* extremely likely */ }
4563	else
4564	{
4565	/* VMRUN failed, shouldn't really happen, Guru. */
4566	if (rc != VINF_SUCCESS)
4567	break;
4568
4569	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4570	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4571	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4572	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4573	break;
4574	}
4575
4576	/* Handle the #VMEXIT. */
4577	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4578	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4579	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4580	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4581	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4582	if (rc == VINF_SUCCESS)
4583	{
4584	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4585	{
4586	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4587	rc = VINF_SVM_VMEXIT;
4588	}
4589	else
4590	{
4591	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4592	continue;
4593	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4594	rc = VINF_EM_RAW_INTERRUPT;
4595	}
4596	}
4597	else
4598	Assert(rc != VINF_SVM_VMEXIT);
4599	break;
4600	/** @todo NSTSVM: handle single-stepping. */
4601	}
4602
4603	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4604	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4605
4606	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4607	return rc;
4608	}
4609	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4610
4611
4612	/**
4613	* Checks if any expensive dtrace probes are enabled and we should go to the
4614	* debug loop.
4615	*
4616	* @returns true if we should use debug loop, false if not.
4617	*/
4618	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4619	{
4620	/* It's probably faster to OR the raw 32-bit counter variables together.
4621	Since the variables are in an array and the probes are next to one
4622	another (more or less), we have good locality. So, better read
4623	eight-nine cache lines ever time and only have one conditional, than
4624	128+ conditionals, right? */
4625	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4626	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4627	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4628	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4629	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4630	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4631	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4632	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4633	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4634	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4635	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4636	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4637	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4638	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4639	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4640	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4641	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4642	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4643	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4644	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4645	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4646	) != 0
4647	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4648	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4649	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4650	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4651	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4652	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4653	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4654	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4655	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4656	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4657	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4658	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4659	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4660	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4661	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4662	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4663	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4664	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4665	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4666	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4673	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4675	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4676	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4677	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4678	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4679	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4680	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4681	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4682	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4683	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4684	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4685	) != 0
4686	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4687	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4688	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4689	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4690	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4691	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4692	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4693	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4694	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4695	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4696	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4697	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4698	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4699	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4700	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4701	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4702	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4703	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4704	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4705	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4713	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4715	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4716	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4717	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4718	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4719	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4720	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4721	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4722	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4723	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4724	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4725	) != 0;
4726	}
4727
4728
4729	/**
4730	* Runs the guest code using AMD-V.
4731	*
4732	* @returns Strict VBox status code.
4733	* @param pVCpu The cross context virtual CPU structure.
4734	*/
4735	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4736	{
4737	AssertPtr(pVCpu);
4738	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4739	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4740	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4741	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4742
4743	uint32_t cLoops = 0;
4744	VBOXSTRICTRC rc;
4745	for (;;)
4746	{
4747	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4748	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4749	#else
4750	NOREF(pCtx);
4751	bool const fInNestedGuestMode = false;
4752	#endif
4753	if (!fInNestedGuestMode)
4754	{
4755	if ( !pVCpu->hm.s.fUseDebugLoop
4756	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4757	&& !DBGFIsStepping(pVCpu)
4758	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledSwBreakpoints)
4759	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4760	else
4761	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4762	}
4763	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4764	else
4765	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4766
4767	if (rc == VINF_SVM_VMRUN)
4768	{
4769	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4770	continue;
4771	}
4772	if (rc == VINF_SVM_VMEXIT)
4773	{
4774	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4775	continue;
4776	}
4777	#endif
4778	break;
4779	}
4780
4781	/* Fixup error codes. */
4782	if (rc == VERR_EM_INTERPRETER)
4783	rc = VINF_EM_RAW_EMULATE_INSTR;
4784	else if (rc == VINF_EM_RESET)
4785	rc = VINF_EM_TRIPLE_FAULT;
4786
4787	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4788	rc = hmR0SvmExitToRing3(pVCpu, rc);
4789	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4790	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4791	return rc;
4792	}
4793
4794	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4795
4796	/**
4797	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4798	*
4799	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4800	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4801	*/
4802	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4803	{
4804	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4805	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4806	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4807	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4808	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4809	const bool fRep = pIoExitInfo->n.u1Rep;
4810	const bool fStrIo = pIoExitInfo->n.u1Str;
4811
4812	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4813	NULL /* pIoExitInfo */);
4814	}
4815
4816
4817	/**
4818	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4819	* SVM_EXIT_INVALID).
4820	*
4821	* @returns VBox status code (informational status codes included).
4822	* @param pVCpu The cross context virtual CPU structure.
4823	* @param pSvmTransient Pointer to the SVM transient structure.
4824	*/
4825	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4826	{
4827	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4828	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4829	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4830
4831	/*
4832	* We import the complete state here because we use separate VMCBs for the guest and the
4833	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4834	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4835	*/
4836	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4837	do { \
4838	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4839	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4840	} while (0)
4841
4842	/*
4843	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4844	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4845	*/
4846	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4847	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4848	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4849	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4850	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4851	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4852
4853	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4854	switch (uExitCode)
4855	{
4856	case SVM_EXIT_CPUID:
4857	{
4858	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4859	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4860	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4861	}
4862
4863	case SVM_EXIT_RDTSC:
4864	{
4865	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4866	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4867	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4868	}
4869
4870	case SVM_EXIT_RDTSCP:
4871	{
4872	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4873	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4874	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4875	}
4876
4877	case SVM_EXIT_MONITOR:
4878	{
4879	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4880	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4881	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4882	}
4883
4884	case SVM_EXIT_MWAIT:
4885	{
4886	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4887	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4888	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4889	}
4890
4891	case SVM_EXIT_HLT:
4892	{
4893	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4894	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4895	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4896	}
4897
4898	case SVM_EXIT_MSR:
4899	{
4900	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4901	{
4902	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4903	uint16_t offMsrpm;
4904	uint8_t uMsrpmBit;
4905	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4906	if (RT_SUCCESS(rc))
4907	{
4908	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4909	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4910
4911	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4912	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4913	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4914
4915	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4916	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4917	{
4918	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4919	}
4920	}
4921	else
4922	{
4923	/*
4924	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4925	* See AMD-V spec. "15.11 MSR Intercepts".
4926	*/
4927	Assert(rc == VERR_OUT_OF_RANGE);
4928	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4929	}
4930	}
4931	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4932	}
4933
4934	case SVM_EXIT_IOIO:
4935	{
4936	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4937	{
4938	SVMIOIOEXITINFO IoExitInfo;
4939	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4940	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4941	if (fIntercept)
4942	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4943	}
4944	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4945	}
4946
4947	case SVM_EXIT_XCPT_PF:
4948	{
4949	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4950	if (pVM->hmr0.s.fNestedPaging)
4951	{
4952	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4953	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4954
4955	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4956	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4957	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4958
4959	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4960	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4961	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4962	return VINF_SUCCESS;
4963	}
4964	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4965	}
4966
4967	case SVM_EXIT_XCPT_UD:
4968	{
4969	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4970	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4971	hmR0SvmSetPendingXcptUD(pVCpu);
4972	return VINF_SUCCESS;
4973	}
4974
4975	case SVM_EXIT_XCPT_MF:
4976	{
4977	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4978	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4979	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4980	}
4981
4982	case SVM_EXIT_XCPT_DB:
4983	{
4984	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4985	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4986	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
4987	}
4988
4989	case SVM_EXIT_XCPT_AC:
4990	{
4991	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
4992	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4993	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
4994	}
4995
4996	case SVM_EXIT_XCPT_BP:
4997	{
4998	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
4999	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5000	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5001	}
5002
5003	case SVM_EXIT_READ_CR0:
5004	case SVM_EXIT_READ_CR3:
5005	case SVM_EXIT_READ_CR4:
5006	{
5007	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5008	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5009	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5010	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5011	}
5012
5013	case SVM_EXIT_CR0_SEL_WRITE:
5014	{
5015	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5016	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5017	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5018	}
5019
5020	case SVM_EXIT_WRITE_CR0:
5021	case SVM_EXIT_WRITE_CR3:
5022	case SVM_EXIT_WRITE_CR4:
5023	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5024	{
5025	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5026	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5027
5028	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5029	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5030	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5031	}
5032
5033	case SVM_EXIT_PAUSE:
5034	{
5035	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5036	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5037	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5038	}
5039
5040	case SVM_EXIT_VINTR:
5041	{
5042	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5043	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5044	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5045	}
5046
5047	case SVM_EXIT_INTR:
5048	case SVM_EXIT_NMI:
5049	case SVM_EXIT_SMI:
5050	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5051	{
5052	/*
5053	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5054	*
5055	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5056	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5057	* nested-guest #VMEXIT.
5058	*
5059	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5060	* while trying to inject interrupts, see comment at the top of this function.
5061	*/
5062	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5063	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5064	}
5065
5066	case SVM_EXIT_FERR_FREEZE:
5067	{
5068	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5069	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5070	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5071	}
5072
5073	case SVM_EXIT_INVLPG:
5074	{
5075	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5076	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5077	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5078	}
5079
5080	case SVM_EXIT_WBINVD:
5081	{
5082	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5083	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5084	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5085	}
5086
5087	case SVM_EXIT_INVD:
5088	{
5089	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5090	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5091	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5092	}
5093
5094	case SVM_EXIT_RDPMC:
5095	{
5096	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5097	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5098	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5099	}
5100
5101	default:
5102	{
5103	switch (uExitCode)
5104	{
5105	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5106	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5107	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5108	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5109	{
5110	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5111	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5112	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5113	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5114	}
5115
5116	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5117	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5118	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5119	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5120	{
5121	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5122	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5123	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5124	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5125	}
5126
5127	case SVM_EXIT_XCPT_DE:
5128	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5129	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5130	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5131	case SVM_EXIT_XCPT_OF:
5132	case SVM_EXIT_XCPT_BR:
5133	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5134	case SVM_EXIT_XCPT_NM:
5135	case SVM_EXIT_XCPT_DF:
5136	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5137	case SVM_EXIT_XCPT_TS:
5138	case SVM_EXIT_XCPT_NP:
5139	case SVM_EXIT_XCPT_SS:
5140	case SVM_EXIT_XCPT_GP:
5141	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5142	case SVM_EXIT_XCPT_15: /* Reserved. */
5143	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5144	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5145	case SVM_EXIT_XCPT_MC:
5146	case SVM_EXIT_XCPT_XF:
5147	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5148	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5149	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5150	{
5151	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5152	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5153	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5154	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5155	}
5156
5157	case SVM_EXIT_XSETBV:
5158	{
5159	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5160	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5161	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5162	}
5163
5164	case SVM_EXIT_TASK_SWITCH:
5165	{
5166	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5167	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5168	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5169	}
5170
5171	case SVM_EXIT_IRET:
5172	{
5173	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5174	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5175	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5176	}
5177
5178	case SVM_EXIT_SHUTDOWN:
5179	{
5180	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5181	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5182	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5183	}
5184
5185	case SVM_EXIT_VMMCALL:
5186	{
5187	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5188	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5189	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5190	}
5191
5192	case SVM_EXIT_CLGI:
5193	{
5194	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5195	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5196	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5197	}
5198
5199	case SVM_EXIT_STGI:
5200	{
5201	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5202	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5203	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5204	}
5205
5206	case SVM_EXIT_VMLOAD:
5207	{
5208	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5209	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5210	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5211	}
5212
5213	case SVM_EXIT_VMSAVE:
5214	{
5215	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5216	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5217	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5218	}
5219
5220	case SVM_EXIT_INVLPGA:
5221	{
5222	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5223	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5224	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5225<