HMSVMR0.cpp@ 103131

Last change on this file since 103131 was 100357, checked in by vboxsync, 15 months ago
Runtime/RTR0MemObj*: Add PhysHighest parameter to RTR0MemObjAllocCont to indicate the maximum allowed physical address for an allocation, bugref:10457 [second attempt]
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 380.3 KB

Line
1	/* $Id: HMSVMR0.cpp 100357 2023-07-04 07:00:26Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_HM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/thread.h>
36
37	#include <VBox/vmm/pdmapi.h>
38	#include <VBox/vmm/dbgf.h>
39	#include <VBox/vmm/iem.h>
40	#include <VBox/vmm/iom.h>
41	#include <VBox/vmm/tm.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/gcm.h>
44	#include <VBox/vmm/gim.h>
45	#include <VBox/vmm/apic.h>
46	#include "HMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include <VBox/err.h>
49	#include "HMSVMR0.h"
50	#include "dtrace/VBoxVMM.h"
51
52	#ifdef DEBUG_ramshankar
53	# define HMSVM_SYNC_FULL_GUEST_STATE
54	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
55	# define HMSVM_ALWAYS_TRAP_PF
56	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
57	#endif
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	#ifdef VBOX_WITH_STATISTICS
64	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
66	if ((u64ExitCode) == SVM_EXIT_NPF) \
67	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
68	else \
69	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
70	} while (0)
71
72	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
74	if ((u64ExitCode) == SVM_EXIT_NPF) \
75	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
76	else \
77	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
78	} while (0)
79
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
81	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
82	if ((u64ExitCode) == SVM_EXIT_NPF) \
83	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
84	else \
85	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
86	} while (0)
87	#else
88	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
89	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
90	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
91	#endif /* !VBOX_WITH_STATISTICS */
92
93	/** If we decide to use a function table approach this can be useful to
94	* switch to a "static DECLCALLBACK(int)". */
95	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
96
97	/**
98	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
99	* guest using hardware-assisted SVM.
100	*
101	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
102	* are swapped and restored across the world-switch and also registers like
103	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
104	* \#VMEXIT.
105	*/
106	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
107	\| CPUMCTX_EXTRN_RFLAGS \
108	\| CPUMCTX_EXTRN_RAX \
109	\| CPUMCTX_EXTRN_RSP \
110	\| CPUMCTX_EXTRN_SREG_MASK \
111	\| CPUMCTX_EXTRN_CR0 \
112	\| CPUMCTX_EXTRN_CR2 \
113	\| CPUMCTX_EXTRN_CR3 \
114	\| CPUMCTX_EXTRN_TABLE_MASK \
115	\| CPUMCTX_EXTRN_DR6 \
116	\| CPUMCTX_EXTRN_DR7 \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_HWVIRT \
121	\| CPUMCTX_EXTRN_INHIBIT_INT \
122	\| CPUMCTX_EXTRN_HM_SVM_MASK)
123
124	/**
125	* Subset of the guest-CPU state that is shared between the guest and host.
126	*/
127	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
128
129	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
130	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
131	do { \
132	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
133	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
134	} while (0)
135
136	/** Assert that the required state bits are fetched. */
137	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
151
152	/** Assert that we're not executing a nested-guest. */
153	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
154	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
155	#else
156	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
157	#endif
158
159	/** Assert that we're executing a nested-guest. */
160	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
161	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
162	#else
163	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
164	#endif
165
166	/** Macro for checking and returning from the using function for
167	* \#VMEXIT intercepts that maybe caused during delivering of another
168	* event in the guest. */
169	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
170	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
171	do \
172	{ \
173	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
174	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
175	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
176	else if ( rc == VINF_EM_RESET \
177	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
178	{ \
179	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
180	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
181	} \
182	else \
183	return rc; \
184	} while (0)
185	#else
186	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
187	do \
188	{ \
189	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
190	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
191	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
192	else \
193	return rc; \
194	} while (0)
195	#endif
196
197	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
198	* instruction that exited. */
199	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
200	do { \
201	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
202	(a_rc) = VINF_EM_DBG_STEPPED; \
203	} while (0)
204
205	/** Validate segment descriptor granularity bit. */
206	#ifdef VBOX_STRICT
207	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
208	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
209	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
210	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
211	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
212	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
213	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
214	#else
215	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
216	#endif
217
218	/**
219	* Exception bitmap mask for all contributory exceptions.
220	*
221	* Page fault is deliberately excluded here as it's conditional as to whether
222	* it's contributory or benign. Page faults are handled separately.
223	*/
224	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
225	\| RT_BIT(X86_XCPT_DE))
226
227	/**
228	* Mandatory/unconditional guest control intercepts.
229	*
230	* SMIs can and do happen in normal operation. We need not intercept them
231	* while executing the guest (or nested-guest).
232	*/
233	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
234	\| SVM_CTRL_INTERCEPT_NMI \
235	\| SVM_CTRL_INTERCEPT_INIT \
236	\| SVM_CTRL_INTERCEPT_RDPMC \
237	\| SVM_CTRL_INTERCEPT_CPUID \
238	\| SVM_CTRL_INTERCEPT_RSM \
239	\| SVM_CTRL_INTERCEPT_HLT \
240	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
241	\| SVM_CTRL_INTERCEPT_MSR_PROT \
242	\| SVM_CTRL_INTERCEPT_INVLPGA \
243	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
244	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
245	\| SVM_CTRL_INTERCEPT_VMRUN \
246	\| SVM_CTRL_INTERCEPT_SKINIT \
247	\| SVM_CTRL_INTERCEPT_WBINVD \
248	\| SVM_CTRL_INTERCEPT_MONITOR \
249	\| SVM_CTRL_INTERCEPT_MWAIT \
250	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
251	\| SVM_CTRL_INTERCEPT_XSETBV)
252
253	/** @name VMCB Clean Bits.
254	*
255	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
256	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
257	* memory.
258	*
259	* @{ */
260	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
261	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
262	/** I/O permission bitmap, MSR permission bitmap. */
263	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
264	/** ASID. */
265	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
266	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
267	V_INTR_VECTOR. */
268	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
269	/** Nested Paging: Nested CR3 (nCR3), PAT. */
270	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
271	/** Control registers (CR0, CR3, CR4, EFER). */
272	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
273	/** Debug registers (DR6, DR7). */
274	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
275	/** GDT, IDT limit and base. */
276	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
277	/** Segment register: CS, SS, DS, ES limit and base. */
278	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
279	/** CR2.*/
280	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
281	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
282	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
283	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
284	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
285	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
286	/** Mask of all valid VMCB Clean bits. */
287	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
288	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
289	\| HMSVM_VMCB_CLEAN_ASID \
290	\| HMSVM_VMCB_CLEAN_INT_CTRL \
291	\| HMSVM_VMCB_CLEAN_NP \
292	\| HMSVM_VMCB_CLEAN_CRX_EFER \
293	\| HMSVM_VMCB_CLEAN_DRX \
294	\| HMSVM_VMCB_CLEAN_DT \
295	\| HMSVM_VMCB_CLEAN_SEG \
296	\| HMSVM_VMCB_CLEAN_CR2 \
297	\| HMSVM_VMCB_CLEAN_LBR \
298	\| HMSVM_VMCB_CLEAN_AVIC)
299	/** @} */
300
301	/**
302	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
303	*/
304	typedef enum SVMMSREXITREAD
305	{
306	/** Reading this MSR causes a \#VMEXIT. */
307	SVMMSREXIT_INTERCEPT_READ = 0xb,
308	/** Reading this MSR does not cause a \#VMEXIT. */
309	SVMMSREXIT_PASSTHRU_READ
310	} SVMMSREXITREAD;
311
312	/**
313	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
314	*/
315	typedef enum SVMMSREXITWRITE
316	{
317	/** Writing to this MSR causes a \#VMEXIT. */
318	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a \#VMEXIT. */
320	SVMMSREXIT_PASSTHRU_WRITE
321	} SVMMSREXITWRITE;
322
323	/**
324	* SVM \#VMEXIT handler.
325	*
326	* @returns Strict VBox status code.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pSvmTransient Pointer to the SVM-transient structure.
329	*/
330	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
331
332
333	/*********************************************************************************************************************************
334	* Internal Functions *
335	*********************************************************************************************************************************/
336	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
337	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
338
339
340	/** @name \#VMEXIT handlers.
341	* @{
342	*/
343	static FNSVMEXITHANDLER hmR0SvmExitIntr;
344	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
345	static FNSVMEXITHANDLER hmR0SvmExitInvd;
346	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
347	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
348	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
349	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
350	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
351	static FNSVMEXITHANDLER hmR0SvmExitHlt;
352	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
353	static FNSVMEXITHANDLER hmR0SvmExitMwait;
354	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
355	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
356	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
357	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
358	static FNSVMEXITHANDLER hmR0SvmExitMsr;
359	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
360	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
361	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
362	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
363	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
364	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
365	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
366	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
367	static FNSVMEXITHANDLER hmR0SvmExitPause;
368	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
369	static FNSVMEXITHANDLER hmR0SvmExitIret;
370	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
371	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
372	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
373	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
374	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
375	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
376	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
377	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
378	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
379	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
380	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
381	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
382	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
383	static FNSVMEXITHANDLER hmR0SvmExitClgi;
384	static FNSVMEXITHANDLER hmR0SvmExitStgi;
385	static FNSVMEXITHANDLER hmR0SvmExitVmload;
386	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
387	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
388	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
389	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
390	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
391	#endif
392	/** @} */
393
394	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
395	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
396	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
397	#endif
398	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
399
400
401	/*********************************************************************************************************************************
402	* Global Variables *
403	*********************************************************************************************************************************/
404	/** Ring-0 memory object for the IO bitmap. */
405	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
406	/** Physical address of the IO bitmap. */
407	static RTHCPHYS g_HCPhysIOBitmap;
408	/** Pointer to the IO bitmap. */
409	static R0PTRTYPE(void *) g_pvIOBitmap;
410
411	#ifdef VBOX_STRICT
412	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
413	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
414	# define HMSVM_LOG_CS RT_BIT_32(2)
415	# define HMSVM_LOG_SS RT_BIT_32(3)
416	# define HMSVM_LOG_FS RT_BIT_32(4)
417	# define HMSVM_LOG_GS RT_BIT_32(5)
418	# define HMSVM_LOG_LBR RT_BIT_32(6)
419	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
420	\| HMSVM_LOG_CR_REGS \
421	\| HMSVM_LOG_CS \
422	\| HMSVM_LOG_SS \
423	\| HMSVM_LOG_FS \
424	\| HMSVM_LOG_GS \
425	\| HMSVM_LOG_LBR)
426
427	/**
428	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
429	*
430	* @param pVCpu The cross context virtual CPU structure.
431	* @param pVmcb Pointer to the VM control block.
432	* @param pszPrefix Log prefix.
433	* @param fFlags Log flags, see HMSVM_LOG_XXX.
434	* @param uVerbose The verbosity level, currently unused.
435	*/
436	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
437	{
438	RT_NOREF2(pVCpu, uVerbose);
439	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
440
441	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
442	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
443
444	if (fFlags & HMSVM_LOG_RBP_RSP)
445	{
446	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
447	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
448	}
449
450	if (fFlags & HMSVM_LOG_CR_REGS)
451	{
452	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
453	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
454	}
455
456	if (fFlags & HMSVM_LOG_CS)
457	{
458	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
459	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
460	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
461	}
462	if (fFlags & HMSVM_LOG_SS)
463	{
464	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
465	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
466	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
467	}
468	if (fFlags & HMSVM_LOG_FS)
469	{
470	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
471	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
472	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
473	}
474	if (fFlags & HMSVM_LOG_GS)
475	{
476	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
477	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
478	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
479	}
480
481	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
482	if (fFlags & HMSVM_LOG_LBR)
483	{
484	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
485	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
486	}
487	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
488	}
489	#endif /* VBOX_STRICT */
490
491
492	/**
493	* Sets up and activates AMD-V on the current CPU.
494	*
495	* @returns VBox status code.
496	* @param pHostCpu The HM physical-CPU structure.
497	* @param pVM The cross context VM structure. Can be
498	* NULL after a resume!
499	* @param pvCpuPage Pointer to the global CPU page.
500	* @param HCPhysCpuPage Physical address of the global CPU page.
501	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
502	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
503	* unused).
504	*/
505	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
506	PCSUPHWVIRTMSRS pHwvirtMsrs)
507	{
508	Assert(!fEnabledByHost);
509	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
510	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
511	Assert(pvCpuPage); NOREF(pvCpuPage);
512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
513
514	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
515
516	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
517	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
518
519	/*
520	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
521	*/
522	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
523	if (u64HostEfer & MSR_K6_EFER_SVME)
524	{
525	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
526	if ( pVM
527	&& pVM->hm.s.svm.fIgnoreInUseError)
528	pHostCpu->fIgnoreAMDVInUseError = true;
529
530	if (!pHostCpu->fIgnoreAMDVInUseError)
531	{
532	ASMSetFlags(fEFlags);
533	return VERR_SVM_IN_USE;
534	}
535	}
536
537	/* Turn on AMD-V in the EFER MSR. */
538	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
539
540	/* Write the physical page address where the CPU will store the host state while executing the VM. */
541	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
542
543	/* Restore interrupts. */
544	ASMSetFlags(fEFlags);
545
546	/*
547	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
548	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
549	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
550	* entirely with before executing any guest code.
551	*/
552	pHostCpu->fFlushAsidBeforeUse = true;
553
554	/*
555	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
556	*/
557	++pHostCpu->cTlbFlushes;
558
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Deactivates AMD-V on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pHostCpu The HM physical-CPU structure.
568	* @param pvCpuPage Pointer to the global CPU page.
569	* @param HCPhysCpuPage Physical address of the global CPU page.
570	*/
571	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
572	{
573	RT_NOREF1(pHostCpu);
574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
575	AssertReturn( HCPhysCpuPage
576	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
577	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
578
579	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
580	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
581
582	/* Turn off AMD-V in the EFER MSR. */
583	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
584	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
585
586	/* Invalidate host state physical address. */
587	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
588
589	/* Restore interrupts. */
590	ASMSetFlags(fEFlags);
591
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Does global AMD-V initialization (called during module initialization).
598	*
599	* @returns VBox status code.
600	*/
601	VMMR0DECL(int) SVMR0GlobalInit(void)
602	{
603	/*
604	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
605	* intercept all IO accesses, it's done once globally here instead of per-VM.
606	*/
607	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
608	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT,
609	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
610	if (RT_FAILURE(rc))
611	return rc;
612
613	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
614	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
615
616	/* Set all bits to intercept all IO accesses. */
617	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
618
619	return VINF_SUCCESS;
620	}
621
622
623	/**
624	* Does global AMD-V termination (called during module termination).
625	*/
626	VMMR0DECL(void) SVMR0GlobalTerm(void)
627	{
628	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
629	{
630	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
631	g_pvIOBitmap = NULL;
632	g_HCPhysIOBitmap = 0;
633	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
634	}
635	}
636
637
638	/**
639	* Frees any allocated per-VCPU structures for a VM.
640	*
641	* @param pVM The cross context VM structure.
642	*/
643	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
644	{
645	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
646	{
647	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
648	AssertPtr(pVCpu);
649
650	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
651	{
652	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
653	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
654	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
655	}
656
657	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
658	{
659	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
660	pVCpu->hmr0.s.svm.pVmcb = NULL;
661	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
662	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
663	}
664
665	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
666	{
667	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
668	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
669	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
670	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
671	}
672	}
673	}
674
675
676	/**
677	* Sets pfnVMRun to the best suited variant.
678	*
679	* This must be called whenever anything changes relative to the SVMR0VMRun
680	* variant selection:
681	* - pVCpu->hm.s.fLoadSaveGuestXcr0
682	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
683	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
684	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
685	* - Perhaps: CPUMCTX.fXStateMask (windows only)
686	*
687	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
688	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
689	*/
690	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
691	{
692	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
693	{
694	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
695	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
696	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
697	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
698	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
699	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
700	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
701	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
702	};
703	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
704	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
705	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
706	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
707	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
708	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
709	}
710
711
712	/**
713	* Selector FNHMSVMVMRUN implementation.
714	*/
715	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
716	{
717	hmR0SvmUpdateVmRunFunction(pVCpu);
718	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
719	}
720
721
722	/**
723	* Does per-VM AMD-V initialization.
724	*
725	* @returns VBox status code.
726	* @param pVM The cross context VM structure.
727	*/
728	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
729	{
730	int rc = VERR_INTERNAL_ERROR_5;
731
732	/*
733	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
734	*/
735	uint32_t u32Family;
736	uint32_t u32Model;
737	uint32_t u32Stepping;
738	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
739	{
740	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
741	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
742	}
743
744	/*
745	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
746	*/
747	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
748	{
749	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
750	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
751	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
752	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
753	}
754
755	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
756	{
757	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
758
759	/*
760	* Initialize the hardware-assisted SVM guest-execution handler.
761	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
762	*/
763	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
764
765	/*
766	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
767	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
768	*/
769	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
770	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
771	if (RT_FAILURE(rc))
772	goto failure_cleanup;
773
774	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
775	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
776	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
777
778	/*
779	* Allocate one page for the guest-state VMCB.
780	*/
781	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
782	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
783	if (RT_FAILURE(rc))
784	goto failure_cleanup;
785
786	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
787	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
788	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
789
790	/*
791	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
792	* SVM to not require one.
793	*/
794	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
795	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
796	if (RT_FAILURE(rc))
797	goto failure_cleanup;
798
799	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
800	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
801	/* Set all bits to intercept all MSR accesses (changed later on). */
802	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
803	}
804
805	return VINF_SUCCESS;
806
807	failure_cleanup:
808	hmR0SvmFreeStructs(pVM);
809	return rc;
810	}
811
812
813	/**
814	* Does per-VM AMD-V termination.
815	*
816	* @returns VBox status code.
817	* @param pVM The cross context VM structure.
818	*/
819	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
820	{
821	hmR0SvmFreeStructs(pVM);
822	return VINF_SUCCESS;
823	}
824
825
826	/**
827	* Returns whether the VMCB Clean Bits feature is supported.
828	*
829	* @returns @c true if supported, @c false otherwise.
830	* @param pVCpu The cross context virtual CPU structure.
831	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
832	*/
833	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
834	{
835	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
836	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
837	if (!fIsNestedGuest)
838	return fHostVmcbCleanBits;
839	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
840	}
841
842
843	/**
844	* Returns whether the decode assists feature is supported.
845	*
846	* @returns @c true if supported, @c false otherwise.
847	* @param pVCpu The cross context virtual CPU structure.
848	*/
849	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
850	{
851	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
852	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
853	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
854	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
855	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
856	#endif
857	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
858	}
859
860
861	/**
862	* Returns whether the NRIP_SAVE feature is supported.
863	*
864	* @returns @c true if supported, @c false otherwise.
865	* @param pVCpu The cross context virtual CPU structure.
866	*/
867	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
868	{
869	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
870	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
871	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
872	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
873	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
874	#endif
875	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
876	}
877
878
879	/**
880	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
881	*
882	* @param pVCpu The cross context virtual CPU structure.
883	* @param pbMsrBitmap Pointer to the MSR bitmap.
884	* @param idMsr The MSR for which the permissions are being set.
885	* @param enmRead MSR read permissions.
886	* @param enmWrite MSR write permissions.
887	*
888	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
889	* caller needs to take care of this.
890	*/
891	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
892	SVMMSREXITWRITE enmWrite)
893	{
894	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
895	uint16_t offMsrpm;
896	uint8_t uMsrpmBit;
897	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
898	AssertRC(rc);
899
900	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
901	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
902
903	pbMsrBitmap += offMsrpm;
904	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
905	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
906	else
907	{
908	if (!fInNestedGuestMode)
909	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
910	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
911	else
912	{
913	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
914	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
915	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
916	else
917	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
918	}
919	#endif
920	}
921
922	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
923	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
924	else
925	{
926	if (!fInNestedGuestMode)
927	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
928	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
929	else
930	{
931	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
932	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
933	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
934	else
935	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
936	}
937	#endif
938	}
939	}
940
941
942	/**
943	* Sets up AMD-V for the specified VM.
944	* This function is only called once per-VM during initalization.
945	*
946	* @returns VBox status code.
947	* @param pVM The cross context VM structure.
948	*/
949	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
950	{
951	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
952	AssertReturn(pVM, VERR_INVALID_PARAMETER);
953
954	/*
955	* Validate and copy over some parameters.
956	*/
957	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
958	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
959	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
960	pVM->hmr0.s.fNestedPaging = fNestedPaging;
961	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
962
963	/*
964	* Determin some configuration parameters.
965	*/
966	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
967	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
968	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
969
970	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
971	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
972
973	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
974	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
975	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
976
977	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
978	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
979	#endif
980
981	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
982	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
983	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
984	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
985
986	/* Always trap #AC for reasons of security. */
987	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
988
989	/* Always trap #DB for reasons of security. */
990	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
991
992	/* Trap exceptions unconditionally (debug purposes). */
993	#ifdef HMSVM_ALWAYS_TRAP_PF
994	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
995	#endif
996	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
997	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
998	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
999	\| RT_BIT_32(X86_XCPT_DE)
1000	\| RT_BIT_32(X86_XCPT_NM)
1001	\| RT_BIT_32(X86_XCPT_UD)
1002	\| RT_BIT_32(X86_XCPT_NP)
1003	\| RT_BIT_32(X86_XCPT_SS)
1004	\| RT_BIT_32(X86_XCPT_GP)
1005	\| RT_BIT_32(X86_XCPT_PF)
1006	\| RT_BIT_32(X86_XCPT_MF)
1007	;
1008	#endif
1009
1010	/* Apply the exceptions intercepts needed by the GIM provider. */
1011	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1012	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1013
1014	/* Apply the exceptions intercepts needed by the GCM fixers. */
1015	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1016	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1017
1018	/* The mesa 3d driver hack needs #GP. */
1019	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1020	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1021
1022	/* Set up unconditional intercepts and conditions. */
1023	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1024	\| SVM_CTRL_INTERCEPT_VMMCALL
1025	\| SVM_CTRL_INTERCEPT_VMSAVE
1026	\| SVM_CTRL_INTERCEPT_VMLOAD
1027	\| SVM_CTRL_INTERCEPT_CLGI
1028	\| SVM_CTRL_INTERCEPT_STGI;
1029
1030	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1031	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1032	#endif
1033
1034	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1035	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1036	{
1037	/* Virtualized VMSAVE/VMLOAD. */
1038	if (fUseVirtVmsaveVmload)
1039	{
1040	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1041	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1042	\| SVM_CTRL_INTERCEPT_VMLOAD);
1043	}
1044	else
1045	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1046
1047	/* Virtual GIF. */
1048	if (fUseVGif)
1049	{
1050	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1051	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1052	\| SVM_CTRL_INTERCEPT_STGI);
1053	}
1054	else
1055	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1056	}
1057	else
1058	#endif
1059	{
1060	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1061	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1062	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1063	}
1064
1065	/* CR4 writes must always be intercepted for tracking PGM mode changes and
1066	AVX (for XCR0 syncing during worlds switching). */
1067	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1068
1069	/* Intercept all DRx reads and writes by default. Changed later on. */
1070	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1071	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1072
1073	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1074	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1075
1076	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1077	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1078	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1079
1080	/* Set the IO permission bitmap physical addresses. */
1081	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1082
1083	/* LBR virtualization. */
1084	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1085
1086	/* The host ASID MBZ, for the guest start with 1. */
1087	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1088
1089	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1090	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1091
1092	/* Without Nested Paging, we need additionally intercepts. */
1093	if (!fNestedPaging)
1094	{
1095	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1096	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1097	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1098
1099	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1100	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1101	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1102
1103	/* Page faults must be intercepted to implement shadow paging. */
1104	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1105	}
1106
1107	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1108	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1109	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1110
1111	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1112	if (fUsePauseFilter)
1113	{
1114	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1115	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1116	if (fPauseFilterThreshold)
1117	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1118	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1119	}
1120
1121	/*
1122	* Setup the MSR permission bitmap.
1123	* The following MSRs are saved/restored automatically during the world-switch.
1124	* Don't intercept guest read/write accesses to these MSRs.
1125	*/
1126	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1127	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1128	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1129	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1130	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1131	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1132	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1133	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1134	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1135	{
1136	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1137	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1138	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1139	}
1140	else
1141	{
1142	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1143	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1144	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1145	}
1146	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1147
1148	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1149	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1150
1151	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1152	{
1153	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1154	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1155	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1156	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1157
1158	/* Copy the VMCB control area. */
1159	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1160
1161	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1162	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1163	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1164	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1165
1166	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1167	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1168
1169	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1170	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1171	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1172	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1173	}
1174
1175	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1176	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1177	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1178	#else
1179	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1180	#endif
1181	return VINF_SUCCESS;
1182	}
1183
1184
1185	/**
1186	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1187	*
1188	* @returns Pointer to the current context VMCB.
1189	* @param pVCpu The cross context virtual CPU structure.
1190	*/
1191	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1192	{
1193	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1194	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1195	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1196	#endif
1197	return pVCpu->hmr0.s.svm.pVmcb;
1198	}
1199
1200
1201	/**
1202	* Gets a pointer to the nested-guest VMCB cache.
1203	*
1204	* @returns Pointer to the nested-guest VMCB cache.
1205	* @param pVCpu The cross context virtual CPU structure.
1206	*/
1207	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1208	{
1209	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1210	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1211	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1212	#else
1213	RT_NOREF(pVCpu);
1214	return NULL;
1215	#endif
1216	}
1217
1218
1219	/**
1220	* Invalidates a guest page by guest virtual address.
1221	*
1222	* @returns VBox status code.
1223	* @param pVCpu The cross context virtual CPU structure.
1224	* @param GCVirt Guest virtual address of the page to invalidate.
1225	*/
1226	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1227	{
1228	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1229
1230	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1231
1232	/* Skip it if a TLB flush is already pending. */
1233	if (!fFlushPending)
1234	{
1235	Log4Func(("%#RGv\n", GCVirt));
1236
1237	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1238	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1239
1240	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1241	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1242	}
1243	return VINF_SUCCESS;
1244	}
1245
1246
1247	/**
1248	* Flushes the appropriate tagged-TLB entries.
1249	*
1250	* @param pHostCpu The HM physical-CPU structure.
1251	* @param pVCpu The cross context virtual CPU structure.
1252	* @param pVmcb Pointer to the VM control block.
1253	*/
1254	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1255	{
1256	/*
1257	* Force a TLB flush for the first world switch if the current CPU differs from the one
1258	* we ran on last. This can happen both for start & resume due to long jumps back to
1259	* ring-3.
1260	*
1261	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1262	* correlation between it and the physical CPU.
1263	*
1264	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1265	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1266	*/
1267	bool fNewAsid = false;
1268	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1269	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1270	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1271	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1272	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1273	#endif
1274	)
1275	{
1276	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1277	pVCpu->hmr0.s.fForceTLBFlush = true;
1278	fNewAsid = true;
1279	}
1280
1281	/* Set TLB flush state as checked until we return from the world switch. */
1282	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1283
1284	/* Check for explicit TLB flushes. */
1285	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1286	{
1287	pVCpu->hmr0.s.fForceTLBFlush = true;
1288	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1289	}
1290
1291	/*
1292	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1293	* This Host CPU requirement takes precedence.
1294	*/
1295	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1296	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1297	{
1298	pHostCpu->uCurrentAsid = 1;
1299	pVCpu->hmr0.s.uCurrentAsid = 1;
1300	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1301	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1302	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1303
1304	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1305	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1306	}
1307	else
1308	{
1309	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1310	if (pVCpu->hmr0.s.fForceTLBFlush)
1311	{
1312	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1313	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1314
1315	if (fNewAsid)
1316	{
1317	++pHostCpu->uCurrentAsid;
1318
1319	bool fHitASIDLimit = false;
1320	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1321	{
1322	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1323	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1324	fHitASIDLimit = true;
1325	}
1326
1327	if ( fHitASIDLimit
1328	\|\| pHostCpu->fFlushAsidBeforeUse)
1329	{
1330	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1331	pHostCpu->fFlushAsidBeforeUse = false;
1332	}
1333
1334	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1335	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1336	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1337	}
1338	else
1339	{
1340	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1341	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1342	else
1343	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1344	}
1345
1346	pVCpu->hmr0.s.fForceTLBFlush = false;
1347	}
1348	}
1349
1350	/* Update VMCB with the ASID. */
1351	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1352	{
1353	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1354	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1355	}
1356
1357	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1358	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1359	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1360	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1361	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1362	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1363	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1364	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1365
1366	#ifdef VBOX_WITH_STATISTICS
1367	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1368	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1369	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1370	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1371	{
1372	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1373	}
1374	else
1375	{
1376	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1377	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1378	}
1379	#endif
1380	}
1381
1382
1383	/**
1384	* Sets an exception intercept in the specified VMCB.
1385	*
1386	* @param pVmcb Pointer to the VM control block.
1387	* @param uXcpt The exception (X86_XCPT_*).
1388	*/
1389	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1390	{
1391	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1392	{
1393	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1394	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1395	}
1396	}
1397
1398
1399	/**
1400	* Clears an exception intercept in the specified VMCB.
1401	*
1402	* @param pVCpu The cross context virtual CPU structure.
1403	* @param pVmcb Pointer to the VM control block.
1404	* @param uXcpt The exception (X86_XCPT_*).
1405	*
1406	* @remarks This takes into account if we're executing a nested-guest and only
1407	* removes the exception intercept if both the guest -and- nested-guest
1408	* are not intercepting it.
1409	*/
1410	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1411	{
1412	Assert(uXcpt != X86_XCPT_DB);
1413	Assert(uXcpt != X86_XCPT_AC);
1414	Assert(uXcpt != X86_XCPT_GP);
1415	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1416	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1417	{
1418	bool fRemove = true;
1419	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1420	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1421	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1422	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1423	{
1424	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1425	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1426	}
1427	# else
1428	RT_NOREF(pVCpu);
1429	# endif
1430	if (fRemove)
1431	{
1432	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1433	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1434	}
1435	}
1436	#else
1437	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1438	#endif
1439	}
1440
1441
1442	/**
1443	* Sets a control intercept in the specified VMCB.
1444	*
1445	* @param pVmcb Pointer to the VM control block.
1446	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1447	*/
1448	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1449	{
1450	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1451	{
1452	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1453	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1454	}
1455	}
1456
1457
1458	/**
1459	* Clears a control intercept in the specified VMCB.
1460	*
1461	* @returns @c true if the intercept is still set, @c false otherwise.
1462	* @param pVCpu The cross context virtual CPU structure.
1463	* @param pVmcb Pointer to the VM control block.
1464	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1465	*
1466	* @remarks This takes into account if we're executing a nested-guest and only
1467	* removes the control intercept if both the guest -and- nested-guest
1468	* are not intercepting it.
1469	*/
1470	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1471	{
1472	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1473	{
1474	bool fRemove = true;
1475	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1476	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1477	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1478	{
1479	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1480	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1481	}
1482	#else
1483	RT_NOREF(pVCpu);
1484	#endif
1485	if (fRemove)
1486	{
1487	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1488	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1489	}
1490	}
1491
1492	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1493	}
1494
1495
1496	/**
1497	* Exports the guest (or nested-guest) CR0 into the VMCB.
1498	*
1499	* @param pVCpu The cross context virtual CPU structure.
1500	* @param pVmcb Pointer to the VM control block.
1501	*
1502	* @remarks This assumes we always pre-load the guest FPU.
1503	* @remarks No-long-jump zone!!!
1504	*/
1505	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1506	{
1507	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1508
1509	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1510	uint64_t const uGuestCr0 = pCtx->cr0;
1511	uint64_t uShadowCr0 = uGuestCr0;
1512
1513	/* Always enable caching. */
1514	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1515
1516	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1517	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1518	{
1519	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1520	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1521	}
1522
1523	/*
1524	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1525	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1526	* see @bugref{7243#c103}.
1527	*/
1528	if (!(uGuestCr0 & X86_CR0_NE))
1529	{
1530	uShadowCr0 \|= X86_CR0_NE;
1531	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1532	}
1533	else
1534	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1535
1536	/*
1537	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1538	*
1539	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1540	* see @bugref{6944}.
1541	*
1542	* We also don't ever want to honor weird things like cache disable from the guest.
1543	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1544	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1545	*/
1546	if (uShadowCr0 == uGuestCr0)
1547	{
1548	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1549	{
1550	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1551	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1552	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1553	}
1554	else
1555	{
1556	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1557	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1558	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1559	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1560	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1561	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1562	}
1563	}
1564	else
1565	{
1566	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1567	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1568	}
1569	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1570
1571	Assert(!RT_HI_U32(uShadowCr0));
1572	if (pVmcb->guest.u64CR0 != uShadowCr0)
1573	{
1574	pVmcb->guest.u64CR0 = uShadowCr0;
1575	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1576	}
1577	}
1578
1579
1580	/**
1581	* Exports the guest (or nested-guest) CR3 into the VMCB.
1582	*
1583	* @param pVCpu The cross context virtual CPU structure.
1584	* @param pVmcb Pointer to the VM control block.
1585	*
1586	* @remarks No-long-jump zone!!!
1587	*/
1588	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1589	{
1590	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1591
1592	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1593	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1594	if (pVM->hmr0.s.fNestedPaging)
1595	{
1596	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1597	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1598	pVmcb->guest.u64CR3 = pCtx->cr3;
1599	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1600	}
1601	else
1602	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1603
1604	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1605	}
1606
1607
1608	/**
1609	* Exports the guest (or nested-guest) CR4 into the VMCB.
1610	*
1611	* @param pVCpu The cross context virtual CPU structure.
1612	* @param pVmcb Pointer to the VM control block.
1613	*
1614	* @remarks No-long-jump zone!!!
1615	*/
1616	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1617	{
1618	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1619
1620	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1621	uint64_t uShadowCr4 = pCtx->cr4;
1622	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1623	{
1624	switch (pVCpu->hm.s.enmShadowMode)
1625	{
1626	case PGMMODE_REAL:
1627	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1628	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1629
1630	case PGMMODE_32_BIT: /* 32-bit paging. */
1631	uShadowCr4 &= ~X86_CR4_PAE;
1632	break;
1633
1634	case PGMMODE_PAE: /* PAE paging. */
1635	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1636	/** Must use PAE paging as we could use physical memory > 4 GB */
1637	uShadowCr4 \|= X86_CR4_PAE;
1638	break;
1639
1640	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1641	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1642	#ifdef VBOX_WITH_64_BITS_GUESTS
1643	break;
1644	#else
1645	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1646	#endif
1647
1648	default: /* shut up gcc */
1649	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1650	}
1651	}
1652
1653	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1654	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1655	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1656	{
1657	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1658	hmR0SvmUpdateVmRunFunction(pVCpu);
1659	}
1660
1661	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1662	if (uShadowCr4 == pCtx->cr4)
1663	{
1664	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1665	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1666	else
1667	{
1668	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1669	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1670	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1671	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1672	}
1673	}
1674	else
1675	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1676
1677	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking
1678	PGM mode changes and AVX (for XCR0 syncing during worlds switching). */
1679	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1680
1681	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1682	Assert(!RT_HI_U32(uShadowCr4));
1683	pVmcb->guest.u64CR4 = uShadowCr4;
1684	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1685
1686	return VINF_SUCCESS;
1687	}
1688
1689
1690	/**
1691	* Exports the guest (or nested-guest) control registers into the VMCB.
1692	*
1693	* @returns VBox status code.
1694	* @param pVCpu The cross context virtual CPU structure.
1695	* @param pVmcb Pointer to the VM control block.
1696	*
1697	* @remarks No-long-jump zone!!!
1698	*/
1699	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1700	{
1701	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1702
1703	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1704	{
1705	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1706	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1707
1708	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1709	{
1710	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1711	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1712	}
1713
1714	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1715	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1716
1717	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1718	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1719	{
1720	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1721	if (RT_FAILURE(rc))
1722	return rc;
1723	}
1724
1725	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1726	}
1727	return VINF_SUCCESS;
1728	}
1729
1730
1731	/**
1732	* Exports the guest (or nested-guest) segment registers into the VMCB.
1733	*
1734	* @param pVCpu The cross context virtual CPU structure.
1735	* @param pVmcb Pointer to the VM control block.
1736	*
1737	* @remarks No-long-jump zone!!!
1738	*/
1739	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1740	{
1741	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1742	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1743
1744	/* Guest segment registers. */
1745	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1746	{
1747	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1748	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1749
1750	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1751	{
1752	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1753	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1754	}
1755
1756	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1757	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1758
1759	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1760	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1761
1762	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1763	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1764
1765	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1766	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1767
1768	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1769	}
1770
1771	/* Guest TR. */
1772	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1773	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1774
1775	/* Guest LDTR. */
1776	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1777	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1778
1779	/* Guest GDTR. */
1780	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1781	{
1782	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1783	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1784	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1785	}
1786
1787	/* Guest IDTR. */
1788	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1789	{
1790	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1791	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1792	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1793	}
1794
1795	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1796	\| HM_CHANGED_GUEST_TABLE_MASK);
1797	}
1798
1799
1800	/**
1801	* Exports the guest (or nested-guest) MSRs into the VMCB.
1802	*
1803	* @param pVCpu The cross context virtual CPU structure.
1804	* @param pVmcb Pointer to the VM control block.
1805	*
1806	* @remarks No-long-jump zone!!!
1807	*/
1808	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1809	{
1810	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1811	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1812
1813	/* Guest Sysenter MSRs. */
1814	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1815	{
1816	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1817	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1818
1819	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1820	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1821
1822	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1823	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1824	}
1825
1826	/*
1827	* Guest EFER MSR.
1828	* AMD-V requires guest EFER.SVME to be set. Weird.
1829	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1830	*/
1831	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1832	{
1833	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1834	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1835	}
1836
1837	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1838	if ( !CPUMIsGuestInLongModeEx(pCtx)
1839	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1840	{
1841	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1842	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1843	}
1844
1845	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1846	{
1847	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1848	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1849	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1850	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1851	}
1852
1853	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1854	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1855
1856	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1857	\| HM_CHANGED_GUEST_EFER_MSR
1858	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1859	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1860
1861	/*
1862	* Setup the PAT MSR (applicable for Nested Paging only).
1863	*
1864	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1865	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1866	*
1867	* While guests can modify and see the modified values through the shadow values,
1868	* we shall not honor any guest modifications of this MSR to ensure caching is always
1869	* enabled similar to how we clear CR0.CD and NW bits.
1870	*
1871	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1872	*/
1873	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1874
1875	/* Enable the last branch record bit if LBR virtualization is enabled. */
1876	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1877	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1878	}
1879
1880
1881	/**
1882	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1883	* the necessary intercepts accordingly.
1884	*
1885	* @param pVCpu The cross context virtual CPU structure.
1886	* @param pVmcb Pointer to the VM control block.
1887	*
1888	* @remarks No-long-jump zone!!!
1889	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1890	*/
1891	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1892	{
1893	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1894
1895	/** @todo Figure out stepping with nested-guest. */
1896	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1897	{
1898	/*
1899	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1900	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1901	* Instead, they are strictly only requested when the nested hypervisor intercepts
1902	* them -- handled while merging VMCB controls.
1903	*
1904	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1905	* then the nested-guest debug state should be actively loaded on the host so that
1906	* nested-guest reads/writes its own debug registers without causing VM-exits.
1907	*/
1908	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1909	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1910	&& !CPUMIsGuestDebugStateActive(pVCpu))
1911	{
1912	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1913	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1914	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1915	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1916	}
1917
1918	pVmcb->guest.u64DR6 = pCtx->dr[6];
1919	pVmcb->guest.u64DR7 = pCtx->dr[7];
1920	return;
1921	}
1922
1923	/*
1924	* Anyone single stepping on the host side? If so, we'll have to use the
1925	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1926	* the VMM level like the VT-x implementations does.
1927	*/
1928	bool fInterceptMovDRx = false;
1929	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1930	if (fStepping)
1931	{
1932	pVCpu->hmr0.s.fClearTrapFlag = true;
1933	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1934	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1935	}
1936
1937	if ( fStepping
1938	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1939	{
1940	/*
1941	* Use the combined guest and host DRx values found in the hypervisor
1942	* register set because the debugger has breakpoints active or someone
1943	* is single stepping on the host side.
1944	*
1945	* Note! DBGF expects a clean DR6 state before executing guest code.
1946	*/
1947	if (!CPUMIsHyperDebugStateActive(pVCpu))
1948	{
1949	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1950	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1951	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1952	}
1953
1954	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1955	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1956	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1957	{
1958	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1959	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1960	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1961	}
1962
1963	/** @todo If we cared, we could optimize to allow the guest to read registers
1964	* with the same values. */
1965	fInterceptMovDRx = true;
1966	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1967	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1968	}
1969	else
1970	{
1971	/*
1972	* Update DR6, DR7 with the guest values if necessary.
1973	*/
1974	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1975	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1976	{
1977	pVmcb->guest.u64DR7 = pCtx->dr[7];
1978	pVmcb->guest.u64DR6 = pCtx->dr[6];
1979	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1980	}
1981	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1982
1983	/*
1984	* If the guest has enabled debug registers, we need to load them prior to
1985	* executing guest code so they'll trigger at the right time.
1986	*/
1987	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1988	{
1989	if (!CPUMIsGuestDebugStateActive(pVCpu))
1990	{
1991	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1992	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1993	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1994	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1995	}
1996	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
1997	}
1998	/*
1999	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2000	* intercept #DB as DR6 is updated in the VMCB.
2001	*
2002	* Note! If we cared and dared, we could skip intercepting \#DB here.
2003	* However, \#DB shouldn't be performance critical, so we'll play safe
2004	* and keep the code similar to the VT-x code and always intercept it.
2005	*/
2006	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2007	fInterceptMovDRx = true;
2008	}
2009
2010	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2011	if (fInterceptMovDRx)
2012	{
2013	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2014	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2015	{
2016	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2017	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2018	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2019	}
2020	}
2021	else
2022	{
2023	if ( pVmcb->ctrl.u16InterceptRdDRx
2024	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2025	{
2026	pVmcb->ctrl.u16InterceptRdDRx = 0;
2027	pVmcb->ctrl.u16InterceptWrDRx = 0;
2028	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2029	}
2030	}
2031	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2032	}
2033
2034	/**
2035	* Exports the hardware virtualization state into the nested-guest
2036	* VMCB.
2037	*
2038	* @param pVCpu The cross context virtual CPU structure.
2039	* @param pVmcb Pointer to the VM control block.
2040	*
2041	* @remarks No-long-jump zone!!!
2042	*/
2043	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2044	{
2045	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2046
2047	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2048	{
2049	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2050	{
2051	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2052	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2053
2054	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2055	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2056	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2057	NOREF(pVM);
2058
2059	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2060	}
2061
2062	/*
2063	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2064	* since SVM doesn't have a preemption timer.
2065	*
2066	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2067	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2068	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2069	*/
2070	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2071	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2072	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2073	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2074	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2075	{
2076	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2077	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2078	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2079	}
2080	else
2081	{
2082	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2083	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2084	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2085	}
2086	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2087
2088	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2089	}
2090	}
2091
2092
2093	/**
2094	* Exports the guest APIC TPR state into the VMCB.
2095	*
2096	* @returns VBox status code.
2097	* @param pVCpu The cross context virtual CPU structure.
2098	* @param pVmcb Pointer to the VM control block.
2099	*/
2100	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2101	{
2102	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2103
2104	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2105	{
2106	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2107	if ( PDMHasApic(pVM)
2108	&& APICIsEnabled(pVCpu))
2109	{
2110	bool fPendingIntr;
2111	uint8_t u8Tpr;
2112	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2113	AssertRCReturn(rc, rc);
2114
2115	/* Assume that we need to trap all TPR accesses and thus need not check on
2116	every #VMEXIT if we should update the TPR. */
2117	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2118	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2119
2120	if (!pVM->hm.s.fTprPatchingActive)
2121	{
2122	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2123	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2124
2125	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2126	can deliver the interrupt to the guest. */
2127	if (fPendingIntr)
2128	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2129	else
2130	{
2131	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2132	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2133	}
2134
2135	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2136	}
2137	else
2138	{
2139	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2140	pVmcb->guest.u64LSTAR = u8Tpr;
2141	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2142
2143	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2144	if (fPendingIntr)
2145	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2146	else
2147	{
2148	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2149	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2150	}
2151	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2152	}
2153	}
2154	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2155	}
2156	return VINF_SUCCESS;
2157	}
2158
2159
2160	/**
2161	* Sets up the exception interrupts required for guest execution in the VMCB.
2162	*
2163	* @param pVCpu The cross context virtual CPU structure.
2164	* @param pVmcb Pointer to the VM control block.
2165	*
2166	* @remarks No-long-jump zone!!!
2167	*/
2168	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2169	{
2170	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2171
2172	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2173	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2174	{
2175	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2176	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2177	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2178	else
2179	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2180
2181	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2182	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2183	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2184	else
2185	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2186
2187	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2188	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2189	}
2190	}
2191
2192
2193	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2194	/**
2195	* Merges guest and nested-guest intercepts for executing the nested-guest using
2196	* hardware-assisted SVM.
2197	*
2198	* This merges the guest and nested-guest intercepts in a way that if the outer
2199	* guest intercept is set we need to intercept it in the nested-guest as
2200	* well.
2201	*
2202	* @param pVCpu The cross context virtual CPU structure.
2203	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2204	*/
2205	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2206	{
2207	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2208	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2209	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2210	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2211
2212	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2213	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2214	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2215
2216	/* Always intercept CR4 writes for tracking PGM mode changes and AVX (for
2217	XCR0 syncing during worlds switching). */
2218	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2219
2220	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2221	if (!pVM->hmr0.s.fNestedPaging)
2222	{
2223	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2224	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2225	}
2226
2227	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2228	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2229	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2230
2231	/*
2232	* Merge the guest's exception intercepts into the nested-guest VMCB.
2233	*
2234	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2235	* while executing the nested-guest.
2236	*
2237	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2238	* be tweaked later depending on how we wish to implement breakpoints.
2239	*
2240	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2241	* loaded into their guests, not ours.
2242	*
2243	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2244	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2245	*/
2246	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2247	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2248	& ~( RT_BIT(X86_XCPT_UD)
2249	\| RT_BIT(X86_XCPT_BP)
2250	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2251	#else
2252	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2253	#endif
2254
2255	/*
2256	* Adjust intercepts while executing the nested-guest that differ from the
2257	* outer guest intercepts.
2258	*
2259	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2260	* that belong to the nested-guest to the outer guest.
2261	*
2262	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2263	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2264	*/
2265	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2266	\| SVM_CTRL_INTERCEPT_VMMCALL))
2267	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2268
2269	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2270	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2271
2272	/* Finally, update the VMCB clean bits. */
2273	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2274	}
2275	#endif
2276
2277
2278	/**
2279	* Enters the AMD-V session.
2280	*
2281	* @returns VBox status code.
2282	* @param pVCpu The cross context virtual CPU structure.
2283	*/
2284	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2285	{
2286	AssertPtr(pVCpu);
2287	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2288	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2289
2290	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2291	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2292	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2293
2294	pVCpu->hmr0.s.fLeaveDone = false;
2295	return VINF_SUCCESS;
2296	}
2297
2298
2299	/**
2300	* Thread-context callback for AMD-V.
2301	*
2302	* This is used together with RTThreadCtxHookCreate() on platforms which
2303	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2304	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2305	*
2306	* @param enmEvent The thread-context event.
2307	* @param pVCpu The cross context virtual CPU structure.
2308	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2309	* @thread EMT(pVCpu)
2310	*/
2311	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2312	{
2313	NOREF(fGlobalInit);
2314
2315	switch (enmEvent)
2316	{
2317	case RTTHREADCTXEVENT_OUT:
2318	{
2319	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2320	VMCPU_ASSERT_EMT(pVCpu);
2321
2322	/* No longjmps (log-flush, locks) in this fragile context. */
2323	VMMRZCallRing3Disable(pVCpu);
2324
2325	if (!pVCpu->hmr0.s.fLeaveDone)
2326	{
2327	hmR0SvmLeave(pVCpu, false /* fImportState */);
2328	pVCpu->hmr0.s.fLeaveDone = true;
2329	}
2330
2331	/* Leave HM context, takes care of local init (term). */
2332	int rc = HMR0LeaveCpu(pVCpu);
2333	AssertRC(rc); NOREF(rc);
2334
2335	/* Restore longjmp state. */
2336	VMMRZCallRing3Enable(pVCpu);
2337	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2338	break;
2339	}
2340
2341	case RTTHREADCTXEVENT_IN:
2342	{
2343	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2344	VMCPU_ASSERT_EMT(pVCpu);
2345
2346	/* No longjmps (log-flush, locks) in this fragile context. */
2347	VMMRZCallRing3Disable(pVCpu);
2348
2349	/*
2350	* Initialize the bare minimum state required for HM. This takes care of
2351	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2352	*/
2353	int rc = hmR0EnterCpu(pVCpu);
2354	AssertRC(rc); NOREF(rc);
2355	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2356	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2357
2358	pVCpu->hmr0.s.fLeaveDone = false;
2359
2360	/* Restore longjmp state. */
2361	VMMRZCallRing3Enable(pVCpu);
2362	break;
2363	}
2364
2365	default:
2366	break;
2367	}
2368	}
2369
2370
2371	/**
2372	* Saves the host state.
2373	*
2374	* @returns VBox status code.
2375	* @param pVCpu The cross context virtual CPU structure.
2376	*
2377	* @remarks No-long-jump zone!!!
2378	*/
2379	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2380	{
2381	NOREF(pVCpu);
2382
2383	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2384	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2385	return VINF_SUCCESS;
2386	}
2387
2388
2389	/**
2390	* Exports the guest or nested-guest state from the virtual-CPU context into the
2391	* VMCB.
2392	*
2393	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2394	* code based on the virtual-CPU mode.
2395	*
2396	* @returns VBox status code.
2397	* @param pVCpu The cross context virtual CPU structure.
2398	* @param pSvmTransient Pointer to the SVM-transient structure.
2399	*
2400	* @remarks No-long-jump zone!!!
2401	*/
2402	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2403	{
2404	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2405
2406	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2407	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2408	Assert(pVmcb);
2409
2410	pVmcb->guest.u64RIP = pCtx->rip;
2411	pVmcb->guest.u64RSP = pCtx->rsp;
2412	pVmcb->guest.u64RFlags = pCtx->eflags.u;
2413	pVmcb->guest.u64RAX = pCtx->rax;
2414
2415	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2416	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2417
2418	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2419	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2420	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2421	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2422	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2423
2424	ASMSetFlags(fEFlags);
2425
2426	if (!fIsNestedGuest)
2427	{
2428	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2429	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2430	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2431	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2432	}
2433
2434	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2435	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2436	\| HM_CHANGED_GUEST_RFLAGS
2437	\| HM_CHANGED_GUEST_GPRS_MASK
2438	\| HM_CHANGED_GUEST_X87
2439	\| HM_CHANGED_GUEST_SSE_AVX
2440	\| HM_CHANGED_GUEST_OTHER_XSAVE
2441	\| HM_CHANGED_GUEST_XCRx
2442	\| HM_CHANGED_GUEST_TSC_AUX
2443	\| HM_CHANGED_GUEST_OTHER_MSRS;
2444	if (fIsNestedGuest)
2445	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2446	\| HM_CHANGED_GUEST_APIC_TPR;
2447
2448	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2449	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2450
2451	#ifdef VBOX_STRICT
2452	/*
2453	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2454	* except for the host-context and/or shared host-guest context bits.
2455	*/
2456	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2457	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2458	("fCtxChanged=%#RX64\n", fCtxChanged));
2459
2460	/*
2461	* If we need to log state that isn't always imported, we'll need to import them here.
2462	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2463	*/
2464	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2465	#endif
2466
2467	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2468	return VINF_SUCCESS;
2469	}
2470
2471	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2472
2473	/**
2474	* Merges the guest and nested-guest MSR permission bitmap.
2475	*
2476	* If the guest is intercepting an MSR we need to intercept it regardless of
2477	* whether the nested-guest is intercepting it or not.
2478	*
2479	* @param pHostCpu The HM physical-CPU structure.
2480	* @param pVCpu The cross context virtual CPU structure.
2481	*
2482	* @remarks No-long-jmp zone!!!
2483	*/
2484	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2485	{
2486	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2487	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2488	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2489
2490	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2491	uint32_t const offRsvdQwords = 0x1800 >> 3;
2492	for (uint32_t i = 0; i < offRsvdQwords; i++)
2493	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2494	}
2495
2496
2497	/**
2498	* Caches the nested-guest VMCB fields before we modify them for execution using
2499	* hardware-assisted SVM.
2500	*
2501	* @returns true if the VMCB was previously already cached, false otherwise.
2502	* @param pVCpu The cross context virtual CPU structure.
2503	*
2504	* @sa HMNotifySvmNstGstVmexit.
2505	*/
2506	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2507	{
2508	/*
2509	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2510	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2511	*
2512	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2513	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2514	*/
2515	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2516	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2517	if (!fWasCached)
2518	{
2519	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2520	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2521	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2522	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2523	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2524	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2525	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2526	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2527	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2528	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2529	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2530	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2531	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2532	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2533	pVmcbNstGstCache->fCacheValid = true;
2534	Log4Func(("Cached VMCB fields\n"));
2535	}
2536
2537	return fWasCached;
2538	}
2539
2540
2541	/**
2542	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2543	*
2544	* This is done the first time we enter nested-guest execution using SVM R0
2545	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2546	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2547	*
2548	* @param pVCpu The cross context virtual CPU structure.
2549	*/
2550	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2551	{
2552	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2553	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2554
2555	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2556
2557	/*
2558	* First cache the nested-guest VMCB fields we may potentially modify.
2559	*/
2560	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2561	if (!fVmcbCached)
2562	{
2563	/*
2564	* The IOPM of the nested-guest can be ignored because the the guest always
2565	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2566	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2567	*/
2568	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2569
2570	/*
2571	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2572	* nested-paging suddenly while executing a VM (see assertion at the end of
2573	* Trap0eHandler() in PGMAllBth.h).
2574	*/
2575	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2576
2577	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2578	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2579
2580	/*
2581	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2582	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2583	*/
2584	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2585
2586	# ifdef DEBUG_ramshankar
2587	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2588	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2589	# endif
2590
2591	/*
2592	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2593	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2594	*/
2595	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2596	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2597	\| SVM_CTRL_INTERCEPT_VMLOAD;
2598
2599	/*
2600	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2601	* CLGI/STGI instructions executed by the nested-guest.
2602	*/
2603	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2604	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2605	\| SVM_CTRL_INTERCEPT_STGI;
2606
2607	/* Merge the guest and nested-guest intercepts. */
2608	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2609
2610	/* Update the VMCB clean bits. */
2611	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2612	}
2613	else
2614	{
2615	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2616	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2617	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2618	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2619	}
2620	}
2621
2622	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2623
2624	/**
2625	* Exports the state shared between the host and guest (or nested-guest) into
2626	* the VMCB.
2627	*
2628	* @param pVCpu The cross context virtual CPU structure.
2629	* @param pVmcb Pointer to the VM control block.
2630	*
2631	* @remarks No-long-jump zone!!!
2632	*/
2633	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2634	{
2635	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2636	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2637
2638	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2639	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2640
2641	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2642	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2643	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2644	}
2645
2646
2647	/**
2648	* Worker for SVMR0ImportStateOnDemand.
2649	*
2650	* @param pVCpu The cross context virtual CPU structure.
2651	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2652	*/
2653	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2654	{
2655	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2656
2657	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2658	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2659	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2660	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2661
2662	/*
2663	* We disable interrupts to make the updating of the state and in particular
2664	* the fExtrn modification atomic wrt to preemption hooks.
2665	*/
2666	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2667
2668	fWhat &= pCtx->fExtrn;
2669	if (fWhat)
2670	{
2671	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2672	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2673	{
2674	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2675	{
2676	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2677	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2678	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2679	}
2680	}
2681
2682	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2683	{
2684	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2685	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2686	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2687	}
2688	#endif
2689
2690	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2691	CPUMUpdateInterruptShadowEx(pCtx, pVmcbCtrl->IntShadow.n.u1IntShadow, pVmcbGuest->u64RIP);
2692
2693	if (fWhat & CPUMCTX_EXTRN_RIP)
2694	pCtx->rip = pVmcbGuest->u64RIP;
2695
2696	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2697	{
2698	pCtx->eflags.u = pVmcbGuest->u64RFlags;
2699	if (pVCpu->hmr0.s.fClearTrapFlag)
2700	{
2701	pVCpu->hmr0.s.fClearTrapFlag = false;
2702	pCtx->eflags.Bits.u1TF = 0;
2703	}
2704	}
2705
2706	if (fWhat & CPUMCTX_EXTRN_RSP)
2707	pCtx->rsp = pVmcbGuest->u64RSP;
2708
2709	if (fWhat & CPUMCTX_EXTRN_RAX)
2710	pCtx->rax = pVmcbGuest->u64RAX;
2711
2712	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2713	{
2714	if (fWhat & CPUMCTX_EXTRN_CS)
2715	{
2716	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2717	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2718	/** @todo SELM might need to be fixed as it too should not care about the
2719	* granularity bit. See @bugref{6785}. */
2720	if ( !pCtx->cs.Attr.n.u1Granularity
2721	&& pCtx->cs.Attr.n.u1Present
2722	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2723	{
2724	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2725	pCtx->cs.Attr.n.u1Granularity = 1;
2726	}
2727	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2728	}
2729	if (fWhat & CPUMCTX_EXTRN_SS)
2730	{
2731	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2732	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2733	/*
2734	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2735	* VMCB and uses that and thus it's possible that when the CPL changes during
2736	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2737	* AMD Fusion CPUs with 64-bit guests.
2738	*
2739	* See AMD spec. 15.5.1 "Basic operation".
2740	*/
2741	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2742	uint8_t const uCpl = pVmcbGuest->u8CPL;
2743	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2744	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2745	}
2746	if (fWhat & CPUMCTX_EXTRN_DS)
2747	{
2748	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2749	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2750	}
2751	if (fWhat & CPUMCTX_EXTRN_ES)
2752	{
2753	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2754	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2755	}
2756	if (fWhat & CPUMCTX_EXTRN_FS)
2757	{
2758	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2759	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2760	}
2761	if (fWhat & CPUMCTX_EXTRN_GS)
2762	{
2763	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2764	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2765	}
2766	}
2767
2768	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2769	{
2770	if (fWhat & CPUMCTX_EXTRN_TR)
2771	{
2772	/*
2773	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2774	* are used between Intel and AMD, see @bugref{6208#c39}.
2775	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2776	*/
2777	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2778	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2779	{
2780	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2781	\|\| CPUMIsGuestInLongModeEx(pCtx))
2782	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2783	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2784	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2785	}
2786	}
2787
2788	if (fWhat & CPUMCTX_EXTRN_LDTR)
2789	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2790
2791	if (fWhat & CPUMCTX_EXTRN_GDTR)
2792	{
2793	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2794	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2795	}
2796
2797	if (fWhat & CPUMCTX_EXTRN_IDTR)
2798	{
2799	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2800	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2801	}
2802	}
2803
2804	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2805	{
2806	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2807	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2808	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2809	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2810	}
2811
2812	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2813	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2814	{
2815	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2816	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2817	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2818	}
2819
2820	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2821	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2822
2823	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2824	{
2825	if (fWhat & CPUMCTX_EXTRN_DR6)
2826	{
2827	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2828	pCtx->dr[6] = pVmcbGuest->u64DR6;
2829	else
2830	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2831	}
2832
2833	if (fWhat & CPUMCTX_EXTRN_DR7)
2834	{
2835	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2836	pCtx->dr[7] = pVmcbGuest->u64DR7;
2837	else
2838	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2839	}
2840	}
2841
2842	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2843	{
2844	if (fWhat & CPUMCTX_EXTRN_CR0)
2845	{
2846	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2847	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2848	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2849	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2850	CPUMSetGuestCR0(pVCpu, uCr0);
2851	VMMRZCallRing3Enable(pVCpu);
2852	}
2853
2854	if (fWhat & CPUMCTX_EXTRN_CR2)
2855	pCtx->cr2 = pVmcbGuest->u64CR2;
2856
2857	if (fWhat & CPUMCTX_EXTRN_CR3)
2858	{
2859	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2860	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2861	{
2862	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2863	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2864	}
2865	}
2866
2867	/* Changes to CR4 are always intercepted. */
2868	}
2869
2870	/* Update fExtrn. */
2871	pCtx->fExtrn &= ~fWhat;
2872
2873	/* If everything has been imported, clear the HM keeper bit. */
2874	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2875	{
2876	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2877	Assert(!pCtx->fExtrn);
2878	}
2879	}
2880	else
2881	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2882
2883	ASMSetFlags(fEFlags);
2884
2885	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2886
2887	/*
2888	* Honor any pending CR3 updates.
2889	*
2890	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2891	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2892	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2893	* handling -> hmR0SvmImportGuestState() and here we are.
2894	*
2895	* The reason for such complicated handling is because VM-exits that call into PGM expect
2896	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2897	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2898	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2899	*
2900	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2901	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2902	*/
2903	if ( VMMRZCallRing3IsEnabled(pVCpu)
2904	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2905	{
2906	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2907	PGMUpdateCR3(pVCpu, pCtx->cr3);
2908	}
2909	}
2910
2911
2912	/**
2913	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2914	* context.
2915	*
2916	* Currently there is no residual state left in the CPU that is not updated in the
2917	* VMCB.
2918	*
2919	* @returns VBox status code.
2920	* @param pVCpu The cross context virtual CPU structure.
2921	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2922	*/
2923	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2924	{
2925	hmR0SvmImportGuestState(pVCpu, fWhat);
2926	return VINF_SUCCESS;
2927	}
2928
2929
2930	/**
2931	* Gets SVM \#VMEXIT auxiliary information.
2932	*
2933	* @returns VBox status code.
2934	* @param pVCpu The cross context virtual CPU structure.
2935	* @param pSvmExitAux Where to store the auxiliary info.
2936	*/
2937	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2938	{
2939	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2940	if (RT_LIKELY(pSvmTransient))
2941	{
2942	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2943	if (RT_LIKELY(pVmcb))
2944	{
2945	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2946	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2947	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2948	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2949	return VINF_SUCCESS;
2950	}
2951	return VERR_SVM_IPE_5;
2952	}
2953	return VERR_NOT_AVAILABLE;
2954	}
2955
2956
2957	/**
2958	* Does the necessary state syncing before returning to ring-3 for any reason
2959	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2960	*
2961	* @param pVCpu The cross context virtual CPU structure.
2962	* @param fImportState Whether to import the guest state from the VMCB back
2963	* to the guest-CPU context.
2964	*
2965	* @remarks No-long-jmp zone!!!
2966	*/
2967	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2968	{
2969	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2970	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2971
2972	/*
2973	* !!! IMPORTANT !!!
2974	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2975	*/
2976
2977	/* Save the guest state if necessary. */
2978	if (fImportState)
2979	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2980
2981	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2982	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2983	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2984
2985	/*
2986	* Restore host debug registers if necessary and resync on next R0 reentry.
2987	*/
2988	#ifdef VBOX_STRICT
2989	if (CPUMIsHyperDebugStateActive(pVCpu))
2990	{
2991	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2992	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2993	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2994	}
2995	#endif
2996	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2997	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2998	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2999
3000	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3001	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3002	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3003	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3004	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3005	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3006	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3007
3008	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3009	}
3010
3011
3012	/**
3013	* Leaves the AMD-V session.
3014	*
3015	* Only used while returning to ring-3 either due to longjump or exits to
3016	* ring-3.
3017	*
3018	* @returns VBox status code.
3019	* @param pVCpu The cross context virtual CPU structure.
3020	*/
3021	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3022	{
3023	HM_DISABLE_PREEMPT(pVCpu);
3024	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3025	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3026
3027	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3028	and done this from the SVMR0ThreadCtxCallback(). */
3029	if (!pVCpu->hmr0.s.fLeaveDone)
3030	{
3031	hmR0SvmLeave(pVCpu, true /* fImportState */);
3032	pVCpu->hmr0.s.fLeaveDone = true;
3033	}
3034
3035	/*
3036	* !!! IMPORTANT !!!
3037	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3038	*/
3039
3040	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3041	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3042	VMMR0ThreadCtxHookDisable(pVCpu);
3043
3044	/* Leave HM context. This takes care of local init (term). */
3045	int rc = HMR0LeaveCpu(pVCpu);
3046
3047	HM_RESTORE_PREEMPT();
3048	return rc;
3049	}
3050
3051
3052	/**
3053	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3054	* any remaining host state) before we go back to ring-3 due to an assertion.
3055	*
3056	* @param pVCpu The cross context virtual CPU structure.
3057	*/
3058	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3059	{
3060	/*
3061	* !!! IMPORTANT !!!
3062	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3063	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3064	*/
3065	VMMR0AssertionRemoveNotification(pVCpu);
3066	VMMRZCallRing3Disable(pVCpu);
3067	HM_DISABLE_PREEMPT(pVCpu);
3068
3069	/* Import the entire guest state. */
3070	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3071
3072	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3073	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3074
3075	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3076	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3077
3078	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3079	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3080	VMMR0ThreadCtxHookDisable(pVCpu);
3081
3082	/* Leave HM context. This takes care of local init (term). */
3083	HMR0LeaveCpu(pVCpu);
3084
3085	HM_RESTORE_PREEMPT();
3086	return VINF_SUCCESS;
3087	}
3088
3089
3090	/**
3091	* Take necessary actions before going back to ring-3.
3092	*
3093	* An action requires us to go back to ring-3. This function does the necessary
3094	* steps before we can safely return to ring-3. This is not the same as longjmps
3095	* to ring-3, this is voluntary.
3096	*
3097	* @returns Strict VBox status code.
3098	* @param pVCpu The cross context virtual CPU structure.
3099	* @param rcExit The reason for exiting to ring-3. Can be
3100	* VINF_VMM_UNKNOWN_RING3_CALL.
3101	*/
3102	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3103	{
3104	Assert(pVCpu);
3105	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3106
3107	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3108	VMMRZCallRing3Disable(pVCpu);
3109	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3110	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3111
3112	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3113	if (pVCpu->hm.s.Event.fPending)
3114	{
3115	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3116	Assert(!pVCpu->hm.s.Event.fPending);
3117	}
3118
3119	/* Sync. the necessary state for going back to ring-3. */
3120	hmR0SvmLeaveSession(pVCpu);
3121	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3122
3123	/* Thread-context hooks are unregistered at this point!!! */
3124	/* Ring-3 callback notifications are unregistered at this point!!! */
3125
3126	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3127	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3128	\| CPUM_CHANGED_LDTR
3129	\| CPUM_CHANGED_GDTR
3130	\| CPUM_CHANGED_IDTR
3131	\| CPUM_CHANGED_TR
3132	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3133	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3134	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3135	{
3136	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3137	}
3138
3139	/* Update the exit-to-ring 3 reason. */
3140	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3141
3142	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3143	if ( rcExit != VINF_EM_RAW_INTERRUPT
3144	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3145	{
3146	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3147	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3148	}
3149
3150	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3151	VMMRZCallRing3Enable(pVCpu);
3152
3153	/*
3154	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3155	* and if we're injecting an event we should have a TRPM trap pending.
3156	*/
3157	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3158	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3159	VERR_SVM_IPE_5);
3160	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3161	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3162	VERR_SVM_IPE_4);
3163
3164	return rcExit;
3165	}
3166
3167
3168	/**
3169	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3170	* intercepts.
3171	*
3172	* @param pVCpu The cross context virtual CPU structure.
3173	* @param pVmcb Pointer to the VM control block.
3174	*
3175	* @remarks No-long-jump zone!!!
3176	*/
3177	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3178	{
3179	/*
3180	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3181	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3182	* RDTSC/RDTSCP as well.
3183	*/
3184	bool fParavirtTsc;
3185	uint64_t uTscOffset;
3186	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3187
3188	bool fIntercept;
3189	if (fCanUseRealTsc)
3190	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3191	else
3192	{
3193	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3194	fIntercept = true;
3195	}
3196
3197	if (!fIntercept)
3198	{
3199	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3200	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3201	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3202	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3203	#endif
3204
3205	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3206	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3207	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3208	}
3209
3210	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3211	information before every VM-entry, hence we have nothing to do here at the moment. */
3212	if (fParavirtTsc)
3213	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3214	}
3215
3216
3217	/**
3218	* Sets an event as a pending event to be injected into the guest.
3219	*
3220	* @param pVCpu The cross context virtual CPU structure.
3221	* @param pEvent Pointer to the SVM event.
3222	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3223	* page-fault.
3224	*
3225	* @remarks Statistics counter assumes this is a guest event being reflected to
3226	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3227	*/
3228	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3229	{
3230	Assert(!pVCpu->hm.s.Event.fPending);
3231	Assert(pEvent->n.u1Valid);
3232
3233	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3234	pVCpu->hm.s.Event.fPending = true;
3235	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3236
3237	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3238	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3239	}
3240
3241
3242	/**
3243	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3244	*
3245	* @param pVCpu The cross context virtual CPU structure.
3246	*/
3247	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3248	{
3249	SVMEVENT Event;
3250	Event.u = 0;
3251	Event.n.u1Valid = 1;
3252	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3253	Event.n.u8Vector = X86_XCPT_DE;
3254	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3255	}
3256
3257
3258	/**
3259	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3260	*
3261	* @param pVCpu The cross context virtual CPU structure.
3262	*/
3263	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3264	{
3265	SVMEVENT Event;
3266	Event.u = 0;
3267	Event.n.u1Valid = 1;
3268	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3269	Event.n.u8Vector = X86_XCPT_UD;
3270	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3271	}
3272
3273
3274	/**
3275	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3276	*
3277	* @param pVCpu The cross context virtual CPU structure.
3278	*/
3279	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3280	{
3281	SVMEVENT Event;
3282	Event.u = 0;
3283	Event.n.u1Valid = 1;
3284	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3285	Event.n.u8Vector = X86_XCPT_DB;
3286	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3287	}
3288
3289
3290	/**
3291	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3292	*
3293	* @param pVCpu The cross context virtual CPU structure.
3294	* @param u32ErrCode The error-code for the page-fault.
3295	* @param uFaultAddress The page fault address (CR2).
3296	*
3297	* @remarks This updates the guest CR2 with @a uFaultAddress!
3298	*/
3299	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3300	{
3301	SVMEVENT Event;
3302	Event.u = 0;
3303	Event.n.u1Valid = 1;
3304	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3305	Event.n.u8Vector = X86_XCPT_PF;
3306	Event.n.u1ErrorCodeValid = 1;
3307	Event.n.u32ErrorCode = u32ErrCode;
3308
3309	/* Update CR2 of the guest. */
3310	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3311	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3312	{
3313	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3314	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3315	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3316	}
3317
3318	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3319	}
3320
3321
3322	/**
3323	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3324	*
3325	* @param pVCpu The cross context virtual CPU structure.
3326	*/
3327	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3328	{
3329	SVMEVENT Event;
3330	Event.u = 0;
3331	Event.n.u1Valid = 1;
3332	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3333	Event.n.u8Vector = X86_XCPT_MF;
3334	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3335	}
3336
3337
3338	/**
3339	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3340	*
3341	* @param pVCpu The cross context virtual CPU structure.
3342	*/
3343	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3344	{
3345	SVMEVENT Event;
3346	Event.u = 0;
3347	Event.n.u1Valid = 1;
3348	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3349	Event.n.u8Vector = X86_XCPT_DF;
3350	Event.n.u1ErrorCodeValid = 1;
3351	Event.n.u32ErrorCode = 0;
3352	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3353	}
3354
3355
3356	/**
3357	* Injects an event into the guest upon VMRUN by updating the relevant field
3358	* in the VMCB.
3359	*
3360	* @param pVCpu The cross context virtual CPU structure.
3361	* @param pVmcb Pointer to the guest VM control block.
3362	* @param pEvent Pointer to the event.
3363	*
3364	* @remarks No-long-jump zone!!!
3365	* @remarks Requires CR0!
3366	*/
3367	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3368	{
3369	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3370	pVmcb->ctrl.EventInject.u = pEvent->u;
3371	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3372	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3373	{
3374	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3375	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3376	}
3377	else
3378	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3379	RT_NOREF(pVCpu);
3380
3381	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3382	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3383	}
3384
3385
3386
3387	/**
3388	* Converts any TRPM trap into a pending HM event. This is typically used when
3389	* entering from ring-3 (not longjmp returns).
3390	*
3391	* @param pVCpu The cross context virtual CPU structure.
3392	*/
3393	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3394	{
3395	Assert(TRPMHasTrap(pVCpu));
3396	Assert(!pVCpu->hm.s.Event.fPending);
3397
3398	uint8_t uVector;
3399	TRPMEVENT enmTrpmEvent;
3400	uint32_t uErrCode;
3401	RTGCUINTPTR GCPtrFaultAddress;
3402	uint8_t cbInstr;
3403
3404	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3405	AssertRC(rc);
3406
3407	SVMEVENT Event;
3408	Event.u = 0;
3409	Event.n.u1Valid = 1;
3410	Event.n.u8Vector = uVector;
3411
3412	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3413	if (enmTrpmEvent == TRPM_TRAP)
3414	{
3415	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3416	switch (uVector)
3417	{
3418	case X86_XCPT_NMI:
3419	{
3420	Event.n.u3Type = SVM_EVENT_NMI;
3421	break;
3422	}
3423
3424	case X86_XCPT_BP:
3425	case X86_XCPT_OF:
3426	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3427	RT_FALL_THRU();
3428
3429	case X86_XCPT_PF:
3430	case X86_XCPT_DF:
3431	case X86_XCPT_TS:
3432	case X86_XCPT_NP:
3433	case X86_XCPT_SS:
3434	case X86_XCPT_GP:
3435	case X86_XCPT_AC:
3436	{
3437	Event.n.u1ErrorCodeValid = 1;
3438	Event.n.u32ErrorCode = uErrCode;
3439	break;
3440	}
3441	}
3442	}
3443	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3444	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3445	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3446	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3447	else
3448	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3449
3450	rc = TRPMResetTrap(pVCpu);
3451	AssertRC(rc);
3452
3453	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3454	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3455
3456	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3457	}
3458
3459
3460	/**
3461	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3462	* AMD-V to execute any instruction.
3463	*
3464	* @param pVCpu The cross context virtual CPU structure.
3465	*/
3466	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3467	{
3468	Assert(pVCpu->hm.s.Event.fPending);
3469	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3470
3471	SVMEVENT Event;
3472	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3473
3474	uint8_t uVector = Event.n.u8Vector;
3475	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3476
3477	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3478
3479	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3480	AssertRC(rc);
3481
3482	if (Event.n.u1ErrorCodeValid)
3483	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3484
3485	if ( enmTrapType == TRPM_TRAP
3486	&& uVector == X86_XCPT_PF)
3487	{
3488	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3489	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3490	}
3491	else if (enmTrapType == TRPM_SOFTWARE_INT)
3492	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3493	pVCpu->hm.s.Event.fPending = false;
3494	}
3495
3496
3497	/**
3498	* Sets the virtual interrupt intercept control in the VMCB.
3499	*
3500	* @param pVCpu The cross context virtual CPU structure.
3501	* @param pVmcb Pointer to the VM control block.
3502	*/
3503	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3504	{
3505	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3506
3507	/*
3508	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3509	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3510	* (updating ISR at the right time) and inject the interrupt.
3511	*
3512	* With AVIC is supported, we could make use of the asynchronously delivery without
3513	* #VMEXIT and we would be passing the AVIC page to SVM.
3514	*
3515	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3516	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3517	*/
3518	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3519	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3520	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3521	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3522	Log4(("Set VINTR intercept\n"));
3523	}
3524
3525
3526	/**
3527	* Clears the virtual interrupt intercept control in the VMCB as
3528	* we are figured the guest is unable process any interrupts
3529	* at this point of time.
3530	*
3531	* @param pVCpu The cross context virtual CPU structure.
3532	* @param pVmcb Pointer to the VM control block.
3533	*/
3534	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3535	{
3536	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3537
3538	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3539	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3540	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3541	{
3542	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3543	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3544	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3545	Log4(("Cleared VINTR intercept\n"));
3546	}
3547	}
3548
3549
3550	/**
3551	* Evaluates the event to be delivered to the guest and sets it as the pending
3552	* event.
3553	*
3554	* @returns Strict VBox status code.
3555	* @param pVCpu The cross context virtual CPU structure.
3556	* @param pSvmTransient Pointer to the SVM transient structure.
3557	*/
3558	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3559	{
3560	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3561	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3562	\| CPUMCTX_EXTRN_RFLAGS
3563	\| CPUMCTX_EXTRN_INHIBIT_INT
3564	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3565
3566	Assert(!pVCpu->hm.s.Event.fPending);
3567	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3568	Assert(pVmcb);
3569
3570	bool const fGif = CPUMGetGuestGif(pCtx);
3571	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(pCtx);
3572	bool const fBlockNmi = CPUMAreInterruptsInhibitedByNmi(pCtx);
3573
3574	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3575	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3576	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3577
3578	/** @todo SMI. SMIs take priority over NMIs. */
3579
3580	/*
3581	* Check if the guest or nested-guest can receive NMIs.
3582	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3583	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3584	*/
3585	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3586	&& !fBlockNmi)
3587	{
3588	if ( fGif
3589	&& !fIntShadow)
3590	{
3591	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3592	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3593	{
3594	Log4(("Intercepting NMI -> #VMEXIT\n"));
3595	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3596	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3597	}
3598	#endif
3599	Log4(("Setting NMI pending for injection\n"));
3600	SVMEVENT Event;
3601	Event.u = 0;
3602	Event.n.u1Valid = 1;
3603	Event.n.u8Vector = X86_XCPT_NMI;
3604	Event.n.u3Type = SVM_EVENT_NMI;
3605	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3606	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3607	}
3608	else if (!fGif)
3609	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3610	else if (!pSvmTransient->fIsNestedGuest)
3611	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3612	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3613	}
3614	/*
3615	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3616	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3617	* it from the APIC device.
3618	*
3619	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3620	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3621	* do that work when we execute nested-guest code esp. since all the required information
3622	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3623	* the virtual interrupt controller.
3624	*
3625	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3626	*/
3627	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3628	&& !pVCpu->hm.s.fSingleInstruction)
3629	{
3630	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u & X86_EFL_IF)
3631	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3632	if ( fGif
3633	&& !fBlockInt
3634	&& !fIntShadow)
3635	{
3636	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3637	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3638	{
3639	Log4(("Intercepting INTR -> #VMEXIT\n"));
3640	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3641	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3642	}
3643	#endif
3644	uint8_t u8Interrupt;
3645	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3646	if (RT_SUCCESS(rc))
3647	{
3648	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3649	SVMEVENT Event;
3650	Event.u = 0;
3651	Event.n.u1Valid = 1;
3652	Event.n.u8Vector = u8Interrupt;
3653	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3654	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3655	}
3656	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3657	{
3658	/*
3659	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3660	* updated eventually when the TPR is written by the guest.
3661	*/
3662	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3663	}
3664	else
3665	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3666	}
3667	else if (!fGif)
3668	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3669	else if (!pSvmTransient->fIsNestedGuest)
3670	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3671	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3672	}
3673
3674	return VINF_SUCCESS;
3675	}
3676
3677
3678	/**
3679	* Injects any pending events into the guest (or nested-guest).
3680	*
3681	* @param pVCpu The cross context virtual CPU structure.
3682	* @param pVmcb Pointer to the VM control block.
3683	*
3684	* @remarks Must only be called when we are guaranteed to enter
3685	* hardware-assisted SVM execution and not return to ring-3
3686	* prematurely.
3687	*/
3688	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3689	{
3690	Assert(!TRPMHasTrap(pVCpu));
3691	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3692
3693	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx);
3694	#ifdef VBOX_STRICT
3695	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3696	bool const fGif = CPUMGetGuestGif(pCtx);
3697	bool fAllowInt = fGif;
3698	if (fGif)
3699	{
3700	/*
3701	* For nested-guests we have no way to determine if we're injecting a physical or
3702	* virtual interrupt at this point. Hence the partial verification below.
3703	*/
3704	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3705	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3706	else
3707	fAllowInt = RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
3708	}
3709	#endif
3710
3711	if (pVCpu->hm.s.Event.fPending)
3712	{
3713	SVMEVENT Event;
3714	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3715	Assert(Event.n.u1Valid);
3716
3717	/*
3718	* Validate event injection pre-conditions.
3719	*/
3720	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3721	{
3722	Assert(fAllowInt);
3723	Assert(!fIntShadow);
3724	}
3725	else if (Event.n.u3Type == SVM_EVENT_NMI)
3726	{
3727	Assert(fGif);
3728	Assert(!fIntShadow);
3729	}
3730
3731	/*
3732	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3733	* do this only when we are surely going to inject the NMI as otherwise if we return
3734	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3735	* SVM R0.
3736	*
3737	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3738	* which will set the VMCS field after actually delivering the NMI which we read on
3739	* VM-exit to determine the state.
3740	*/
3741	if ( Event.n.u3Type == SVM_EVENT_NMI
3742	&& Event.n.u8Vector == X86_XCPT_NMI)
3743	CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3744
3745	/*
3746	* Inject it (update VMCB for injection by the hardware).
3747	*/
3748	Log4(("Injecting pending HM event\n"));
3749	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3750	pVCpu->hm.s.Event.fPending = false;
3751
3752	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3753	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3754	else
3755	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3756	}
3757	else
3758	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3759
3760	/*
3761	* We could have injected an NMI through IEM and continue guest execution using
3762	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3763	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3764	*/
3765	if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
3766	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3767
3768	/*
3769	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3770	*
3771	* For nested-guests: We need to update it too for the scenario where IEM executes
3772	* the nested-guest but execution later continues here with an interrupt shadow active.
3773	*/
3774	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3775	}
3776
3777
3778	/**
3779	* Reports world-switch error and dumps some useful debug info.
3780	*
3781	* @param pVCpu The cross context virtual CPU structure.
3782	* @param rcVMRun The return code from VMRUN (or
3783	* VERR_SVM_INVALID_GUEST_STATE for invalid
3784	* guest-state).
3785	*/
3786	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3787	{
3788	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3789	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3790	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3791
3792	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3793	{
3794	#ifdef VBOX_STRICT
3795	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3796	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3797	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3798	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3799	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3800	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3801	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3802	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3803	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3804	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3805	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3806	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3807
3808	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3809	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3810	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3811
3812	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3813	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3814	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3815	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3816	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3817	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3818	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3819	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3820	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3821	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3822	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3823	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3824
3825	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3826	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3827	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3828	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3829	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3830	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3831	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3832	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3833	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3834	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3835	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3836	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3837	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3838	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3839	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3840	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3841	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3842	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3843	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3844	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3845
3846	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3847
3848	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3849	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3850
3851	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3852	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3853	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3854	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3855	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3856	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3857	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3858	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3859	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3860	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3861	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3862	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3863	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3864	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3865	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3866	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3867	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3868	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3869	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3870	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3871
3872	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3873	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3874
3875	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3876	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3877	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3878	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3879
3880	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3881	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3882
3883	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3884	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3885	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3886	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3887
3888	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3889	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3890	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3891	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3892	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3893	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3894	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3895
3896	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3897	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3898	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3899	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3900
3901	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3902	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3903	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3904
3905	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3906	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3907	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3908	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3909	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3910	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3911	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3912	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3913	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3914	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3915	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3916	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3917
3918	NOREF(pVmcb);
3919	#endif /* VBOX_STRICT */
3920	}
3921	else
3922	Log4Func(("rcVMRun=%d\n", rcVMRun));
3923	}
3924
3925
3926	/**
3927	* Check per-VM and per-VCPU force flag actions that require us to go back to
3928	* ring-3 for one reason or another.
3929	*
3930	* @returns Strict VBox status code (information status code included).
3931	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3932	* ring-3.
3933	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3934	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3935	* interrupts)
3936	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3937	* all EMTs to be in ring-3.
3938	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3939	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3940	* to the EM loop.
3941	*
3942	* @param pVCpu The cross context virtual CPU structure.
3943	*/
3944	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3945	{
3946	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3947
3948	/* Could happen as a result of longjump. */
3949	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3950	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3951
3952	/* Update pending interrupts into the APIC's IRR. */
3953	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3954	APICUpdatePendingInterrupts(pVCpu);
3955
3956	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3957	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3958	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3959	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3960	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3961	{
3962	/* Pending PGM C3 sync. */
3963	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3964	{
3965	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3966	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3967	if (rc != VINF_SUCCESS)
3968	{
3969	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3970	return rc;
3971	}
3972	}
3973
3974	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3975	/* -XXX- what was that about single stepping? */
3976	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3977	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3978	{
3979	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3980	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3981	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3982	return rc;
3983	}
3984
3985	/* Pending VM request packets, such as hardware interrupts. */
3986	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3987	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3988	{
3989	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
3990	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3991	return VINF_EM_PENDING_REQUEST;
3992	}
3993
3994	/* Pending PGM pool flushes. */
3995	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3996	{
3997	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
3998	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
3999	return VINF_PGM_POOL_FLUSH_PENDING;
4000	}
4001
4002	/* Pending DMA requests. */
4003	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4004	{
4005	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4006	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4007	return VINF_EM_RAW_TO_R3;
4008	}
4009	}
4010
4011	return VINF_SUCCESS;
4012	}
4013
4014
4015	/**
4016	* Does the preparations before executing guest code in AMD-V.
4017	*
4018	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4019	* recompiler. We must be cautious what we do here regarding committing
4020	* guest-state information into the VMCB assuming we assuredly execute the guest
4021	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4022	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4023	* that the recompiler can (and should) use them when it resumes guest
4024	* execution. Otherwise such operations must be done when we can no longer
4025	* exit to ring-3.
4026	*
4027	* @returns Strict VBox status code (informational status codes included).
4028	* @retval VINF_SUCCESS if we can proceed with running the guest.
4029	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4030	*
4031	* @param pVCpu The cross context virtual CPU structure.
4032	* @param pSvmTransient Pointer to the SVM transient structure.
4033	*/
4034	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4035	{
4036	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4037
4038	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4039	if (pSvmTransient->fIsNestedGuest)
4040	{
4041	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4042	return VINF_EM_RESCHEDULE_REM;
4043	}
4044	#endif
4045
4046	/* Check force flag actions that might require us to go back to ring-3. */
4047	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4048	if (rc != VINF_SUCCESS)
4049	return rc;
4050
4051	if (TRPMHasTrap(pVCpu))
4052	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4053	else if (!pVCpu->hm.s.Event.fPending)
4054	{
4055	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4056	if ( rc != VINF_SUCCESS
4057	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4058	{
4059	/* If a nested-guest VM-exit occurred, bail. */
4060	if (pSvmTransient->fIsNestedGuest)
4061	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4062	return rc;
4063	}
4064	}
4065
4066	/*
4067	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4068	* Just do it in software, see @bugref{8411}.
4069	* NB: If we could continue a task switch exit we wouldn't need to do this.
4070	*/
4071	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4072	if (RT_UNLIKELY( !g_fHmSvmFeatures
4073	&& pVCpu->hm.s.Event.fPending
4074	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4075	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4076
4077	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4078	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4079	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4080	#endif
4081
4082	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4083	/*
4084	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4085	*/
4086	if (pSvmTransient->fIsNestedGuest)
4087	hmR0SvmSetupVmcbNested(pVCpu);
4088	#endif
4089
4090	/*
4091	* Export the guest state bits that are not shared with the host in any way as we can
4092	* longjmp or get preempted in the midst of exporting some of the state.
4093	*/
4094	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4095	AssertRCReturn(rc, rc);
4096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4097
4098	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4099	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4100
4101	/*
4102	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4103	* world-switch so we can update it on the way back if the guest changed the TPR.
4104	*/
4105	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4106	{
4107	Assert(!pSvmTransient->fIsNestedGuest);
4108	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4109	if (pVM->hm.s.fTprPatchingActive)
4110	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4111	else
4112	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4113	}
4114
4115	/*
4116	* No longjmps to ring-3 from this point on!!!
4117	*
4118	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4119	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4120	*/
4121	VMMRZCallRing3Disable(pVCpu);
4122
4123	/*
4124	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4125	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4126	* preemption disabled for a while. Since this is purly to aid the
4127	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4128	* disable interrupt on NT.
4129	*
4130	* We need to check for force-flags that could've possible been altered since we last
4131	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4132	* see @bugref{6398}).
4133	*
4134	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4135	* to ring-3 before executing guest code.
4136	*/
4137	pSvmTransient->fEFlags = ASMIntDisableFlags();
4138	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4139	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4140	{
4141	ASMSetFlags(pSvmTransient->fEFlags);
4142	VMMRZCallRing3Enable(pVCpu);
4143	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4144	return VINF_EM_RAW_TO_R3;
4145	}
4146	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4147	{
4148	ASMSetFlags(pSvmTransient->fEFlags);
4149	VMMRZCallRing3Enable(pVCpu);
4150	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4151	return VINF_EM_RAW_INTERRUPT;
4152	}
4153
4154	return VINF_SUCCESS;
4155	}
4156
4157
4158	/**
4159	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4160	* doing so.
4161	*
4162	* This means there is no backing out to ring-3 or anywhere else at this point.
4163	*
4164	* @param pVCpu The cross context virtual CPU structure.
4165	* @param pSvmTransient Pointer to the SVM transient structure.
4166	*
4167	* @remarks Called with preemption disabled.
4168	* @remarks No-long-jump zone!!!
4169	*/
4170	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4171	{
4172	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4173	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4174
4175	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4176	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4177
4178	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4179	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4180
4181	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4182
4183	if (!CPUMIsGuestFPUStateActive(pVCpu))
4184	{
4185	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4186	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4187	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4188	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4189	}
4190
4191	/* Load the state shared between host and guest (FPU, debug). */
4192	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4193	hmR0SvmExportSharedState(pVCpu, pVmcb);
4194
4195	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4196	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4197
4198	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4199	RTCPUID const idHostCpu = pHostCpu->idCpu;
4200	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4201
4202	/* Setup TSC offsetting. */
4203	if ( pSvmTransient->fUpdateTscOffsetting
4204	\|\| fMigratedHostCpu)
4205	{
4206	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4207	pSvmTransient->fUpdateTscOffsetting = false;
4208	}
4209
4210	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4211	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4212	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4213	else
4214	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4215
4216	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4217	if (fMigratedHostCpu)
4218	pVmcb->ctrl.u32VmcbCleanBits = 0;
4219
4220	/* Store status of the shared guest-host state at the time of VMRUN. */
4221	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4222	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4223
4224	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4225	uint8_t *pbMsrBitmap;
4226	if (!pSvmTransient->fIsNestedGuest)
4227	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4228	else
4229	{
4230	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4231	* MSRPM and only perform this if it changed also use EVEX.POR when it
4232	* does. */
4233	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4234
4235	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4236	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4237	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4238	}
4239	#else
4240	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4241	#endif
4242
4243	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4244	/* Flush the appropriate tagged-TLB entries. */
4245	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4246	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4247
4248	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4249
4250	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4251	to start executing. */
4252
4253	/*
4254	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4255	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4256	*
4257	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4258	*/
4259	if ( g_CpumHostFeatures.s.fRdTscP
4260	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4261	{
4262	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4263	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4264	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4265	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4266	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4267	pSvmTransient->fRestoreTscAuxMsr = true;
4268	}
4269	else
4270	{
4271	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4272	pSvmTransient->fRestoreTscAuxMsr = false;
4273	}
4274	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4275
4276	/*
4277	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4278	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4279	* from the VMCB.
4280	*/
4281	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4282	if (!fSupportsVmcbCleanBits)
4283	pVmcb->ctrl.u32VmcbCleanBits = 0;
4284	}
4285
4286
4287	/**
4288	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4289	*
4290	* @returns VBox strict status code.
4291	* @param pVCpu The cross context virtual CPU structure.
4292	* @param HCPhysVmcb The host physical address of the VMCB.
4293	*
4294	* @remarks No-long-jump zone!!!
4295	*/
4296	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4297	{
4298	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4299	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4300	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4301	}
4302
4303
4304	/**
4305	* Performs some essential restoration of state after running guest (or
4306	* nested-guest) code in AMD-V.
4307	*
4308	* @param pVCpu The cross context virtual CPU structure.
4309	* @param pSvmTransient Pointer to the SVM transient structure.
4310	* @param rcVMRun Return code of VMRUN.
4311	*
4312	* @remarks Called with interrupts disabled.
4313	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4314	* unconditionally when it is safe to do so.
4315	*/
4316	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4317	{
4318	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4319
4320	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4321	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4322
4323	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4324	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4325
4326	/* TSC read must be done early for maximum accuracy. */
4327	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4328	{
4329	if (!pSvmTransient->fIsNestedGuest)
4330	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4331	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4332	else
4333	{
4334	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4335	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4336	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4337	}
4338	#endif
4339	}
4340
4341	if (pSvmTransient->fRestoreTscAuxMsr)
4342	{
4343	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4344	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4345	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4346	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4347	}
4348
4349	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4350	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4351	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4352	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4353
4354	Assert(!(ASMGetFlags() & X86_EFL_IF));
4355	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4356	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4357
4358	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4359	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4360	{
4361	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4362	return;
4363	}
4364
4365	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4366	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4367	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4368	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4369
4370	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4371	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4372	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4373	#else
4374	/*
4375	* Always import the following:
4376	*
4377	* - RIP for exit optimizations and evaluating event injection on re-entry.
4378	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4379	* state on preemption.
4380	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4381	* - CS for exit optimizations.
4382	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4383	* assembly switcher code.
4384	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4385	*/
4386	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4387	\| CPUMCTX_EXTRN_RFLAGS
4388	\| CPUMCTX_EXTRN_RAX
4389	\| CPUMCTX_EXTRN_RSP
4390	\| CPUMCTX_EXTRN_CS
4391	\| CPUMCTX_EXTRN_HWVIRT
4392	\| CPUMCTX_EXTRN_INHIBIT_INT
4393	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4394	\| HMSVM_CPUMCTX_SHARED_STATE);
4395	#endif
4396
4397	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4398	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4399	{
4400	Assert(!pSvmTransient->fIsNestedGuest);
4401	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4402	if ( pVM->hm.s.fTprPatchingActive
4403	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4404	{
4405	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4406	AssertRC(rc);
4407	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4408	}
4409	/* Sync TPR when we aren't intercepting CR8 writes. */
4410	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4411	{
4412	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4413	AssertRC(rc);
4414	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4415	}
4416	}
4417
4418	#ifdef DEBUG_ramshankar
4419	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4420	{
4421	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4422	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4423	0 /* uVerbose */);
4424	}
4425	#endif
4426
4427	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4428	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4429	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4430	}
4431
4432
4433	/**
4434	* Runs the guest code using AMD-V.
4435	*
4436	* @returns Strict VBox status code.
4437	* @param pVCpu The cross context virtual CPU structure.
4438	* @param pcLoops Pointer to the number of executed loops.
4439	*/
4440	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4441	{
4442	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4443	Assert(pcLoops);
4444	Assert(*pcLoops <= cMaxResumeLoops);
4445
4446	SVMTRANSIENT SvmTransient;
4447	RT_ZERO(SvmTransient);
4448	SvmTransient.fUpdateTscOffsetting = true;
4449	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4450
4451	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4452	for (;;)
4453	{
4454	Assert(!HMR0SuspendPending());
4455	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4456
4457	/* Preparatory work for running nested-guest code, this may force us to return to
4458	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4459	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4460	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4461	if (rc != VINF_SUCCESS)
4462	break;
4463
4464	/*
4465	* No longjmps to ring-3 from this point on!!!
4466	*
4467	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4468	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4469	*/
4470	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4471	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4472
4473	/* Restore any residual host-state and save any bits shared between host and guest
4474	into the guest-CPU state. Re-enables interrupts! */
4475	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4476
4477	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4478	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4479	{
4480	if (rc == VINF_SUCCESS)
4481	rc = VERR_SVM_INVALID_GUEST_STATE;
4482	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4483	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4484	break;
4485	}
4486
4487	/* Handle the #VMEXIT. */
4488	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4489	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4490	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4491	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4492	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4493	if (rc != VINF_SUCCESS)
4494	break;
4495	if (++(*pcLoops) >= cMaxResumeLoops)
4496	{
4497	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4498	rc = VINF_EM_RAW_INTERRUPT;
4499	break;
4500	}
4501	}
4502
4503	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4504	return rc;
4505	}
4506
4507
4508	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4509	/**
4510	* Runs the nested-guest code using AMD-V.
4511	*
4512	* @returns Strict VBox status code.
4513	* @param pVCpu The cross context virtual CPU structure.
4514	* @param pcLoops Pointer to the number of executed loops. If we're switching
4515	* from the guest-code execution loop to this nested-guest
4516	* execution loop pass the remainder value, else pass 0.
4517	*/
4518	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4519	{
4520	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4521	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4522	Assert(pcLoops);
4523	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4524	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4525	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4526
4527	SVMTRANSIENT SvmTransient;
4528	RT_ZERO(SvmTransient);
4529	SvmTransient.fUpdateTscOffsetting = true;
4530	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4531	SvmTransient.fIsNestedGuest = true;
4532
4533	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4534	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4535
4536	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4537	for (;;)
4538	{
4539	Assert(!HMR0SuspendPending());
4540	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4541
4542	/* Preparatory work for running nested-guest code, this may force us to return to
4543	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4544	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4545	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4546	if ( rc != VINF_SUCCESS
4547	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4548	break;
4549
4550	/*
4551	* No longjmps to ring-3 from this point on!!!
4552	*
4553	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4554	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4555	*/
4556	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4557
4558	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4559
4560	/* Restore any residual host-state and save any bits shared between host and guest
4561	into the guest-CPU state. Re-enables interrupts! */
4562	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4563
4564	if (RT_LIKELY( rc == VINF_SUCCESS
4565	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4566	{ /* extremely likely */ }
4567	else
4568	{
4569	/* VMRUN failed, shouldn't really happen, Guru. */
4570	if (rc != VINF_SUCCESS)
4571	break;
4572
4573	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4574	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4575	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4576	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4577	break;
4578	}
4579
4580	/* Handle the #VMEXIT. */
4581	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4582	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4583	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4584	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4585	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4586	if (rc == VINF_SUCCESS)
4587	{
4588	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4589	{
4590	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4591	rc = VINF_SVM_VMEXIT;
4592	}
4593	else
4594	{
4595	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4596	continue;
4597	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4598	rc = VINF_EM_RAW_INTERRUPT;
4599	}
4600	}
4601	else
4602	Assert(rc != VINF_SVM_VMEXIT);
4603	break;
4604	/** @todo NSTSVM: handle single-stepping. */
4605	}
4606
4607	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4608	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4609
4610	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4611	return rc;
4612	}
4613	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4614
4615
4616	/**
4617	* Checks if any expensive dtrace probes are enabled and we should go to the
4618	* debug loop.
4619	*
4620	* @returns true if we should use debug loop, false if not.
4621	*/
4622	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4623	{
4624	/* It's probably faster to OR the raw 32-bit counter variables together.
4625	Since the variables are in an array and the probes are next to one
4626	another (more or less), we have good locality. So, better read
4627	eight-nine cache lines ever time and only have one conditional, than
4628	128+ conditionals, right? */
4629	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4630	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4631	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4632	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4633	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4634	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4635	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4636	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4637	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4638	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4639	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4640	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4641	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4642	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4643	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4644	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4645	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4646	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4647	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4648	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4649	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4650	) != 0
4651	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4652	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4653	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4654	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4655	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4656	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4657	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4658	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4659	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4660	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4661	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4662	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4663	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4664	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4665	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4666	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4673	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4675	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4676	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4677	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4678	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4679	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4680	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4681	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4682	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4683	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4684	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4685	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4686	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4687	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4689	) != 0
4690	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4691	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4692	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4693	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4694	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4695	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4696	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4697	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4698	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4699	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4700	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4701	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4702	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4703	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4704	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4705	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4713	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4715	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4716	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4717	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4718	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4719	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4720	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4721	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4722	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4723	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4724	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4725	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4726	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4727	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4729	) != 0;
4730	}
4731
4732
4733	/**
4734	* Runs the guest code using AMD-V.
4735	*
4736	* @returns Strict VBox status code.
4737	* @param pVCpu The cross context virtual CPU structure.
4738	*/
4739	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4740	{
4741	AssertPtr(pVCpu);
4742	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4743	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4744	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4745	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4746
4747	uint32_t cLoops = 0;
4748	VBOXSTRICTRC rc;
4749	for (;;)
4750	{
4751	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4752	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4753	#else
4754	NOREF(pCtx);
4755	bool const fInNestedGuestMode = false;
4756	#endif
4757	if (!fInNestedGuestMode)
4758	{
4759	if ( !pVCpu->hm.s.fUseDebugLoop
4760	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4761	&& !DBGFIsStepping(pVCpu)
4762	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4763	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4764	else
4765	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4766	}
4767	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4768	else
4769	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4770
4771	if (rc == VINF_SVM_VMRUN)
4772	{
4773	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4774	continue;
4775	}
4776	if (rc == VINF_SVM_VMEXIT)
4777	{
4778	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4779	continue;
4780	}
4781	#endif
4782	break;
4783	}
4784
4785	/* Fixup error codes. */
4786	if (rc == VERR_EM_INTERPRETER)
4787	rc = VINF_EM_RAW_EMULATE_INSTR;
4788	else if (rc == VINF_EM_RESET)
4789	rc = VINF_EM_TRIPLE_FAULT;
4790
4791	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4792	rc = hmR0SvmExitToRing3(pVCpu, rc);
4793	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4794	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4795	return rc;
4796	}
4797
4798	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4799
4800	/**
4801	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4802	*
4803	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4804	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4805	*/
4806	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4807	{
4808	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4809	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4810	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4811	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4812	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4813	const bool fRep = pIoExitInfo->n.u1Rep;
4814	const bool fStrIo = pIoExitInfo->n.u1Str;
4815
4816	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4817	NULL /* pIoExitInfo */);
4818	}
4819
4820
4821	/**
4822	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4823	* SVM_EXIT_INVALID).
4824	*
4825	* @returns VBox status code (informational status codes included).
4826	* @param pVCpu The cross context virtual CPU structure.
4827	* @param pSvmTransient Pointer to the SVM transient structure.
4828	*/
4829	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4830	{
4831	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4832	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4833	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4834
4835	/*
4836	* We import the complete state here because we use separate VMCBs for the guest and the
4837	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4838	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4839	*/
4840	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4841	do { \
4842	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4843	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4844	} while (0)
4845
4846	/*
4847	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4848	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4849	*/
4850	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4851	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4852	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4853	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4854	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4855	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4856
4857	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4858	switch (uExitCode)
4859	{
4860	case SVM_EXIT_CPUID:
4861	{
4862	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4863	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4864	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4865	}
4866
4867	case SVM_EXIT_RDTSC:
4868	{
4869	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4870	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4871	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4872	}
4873
4874	case SVM_EXIT_RDTSCP:
4875	{
4876	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4877	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4878	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4879	}
4880
4881	case SVM_EXIT_MONITOR:
4882	{
4883	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4884	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4885	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4886	}
4887
4888	case SVM_EXIT_MWAIT:
4889	{
4890	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4891	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4892	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4893	}
4894
4895	case SVM_EXIT_HLT:
4896	{
4897	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4898	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4899	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4900	}
4901
4902	case SVM_EXIT_MSR:
4903	{
4904	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4905	{
4906	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4907	uint16_t offMsrpm;
4908	uint8_t uMsrpmBit;
4909	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4910	if (RT_SUCCESS(rc))
4911	{
4912	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4913	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4914
4915	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4916	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4917	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4918
4919	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4920	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4921	{
4922	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4923	}
4924	}
4925	else
4926	{
4927	/*
4928	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4929	* See AMD-V spec. "15.11 MSR Intercepts".
4930	*/
4931	Assert(rc == VERR_OUT_OF_RANGE);
4932	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4933	}
4934	}
4935	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4936	}
4937
4938	case SVM_EXIT_IOIO:
4939	{
4940	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4941	{
4942	SVMIOIOEXITINFO IoExitInfo;
4943	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4944	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4945	if (fIntercept)
4946	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4947	}
4948	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4949	}
4950
4951	case SVM_EXIT_XCPT_PF:
4952	{
4953	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4954	if (pVM->hmr0.s.fNestedPaging)
4955	{
4956	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4957	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4958
4959	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4960	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4961	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4962
4963	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4964	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4965	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4966	return VINF_SUCCESS;
4967	}
4968	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4969	}
4970
4971	case SVM_EXIT_XCPT_UD:
4972	{
4973	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4974	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4975	hmR0SvmSetPendingXcptUD(pVCpu);
4976	return VINF_SUCCESS;
4977	}
4978
4979	case SVM_EXIT_XCPT_MF:
4980	{
4981	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4982	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4983	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4984	}
4985
4986	case SVM_EXIT_XCPT_DB:
4987	{
4988	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4989	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4990	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
4991	}
4992
4993	case SVM_EXIT_XCPT_AC:
4994	{
4995	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
4996	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4997	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
4998	}
4999
5000	case SVM_EXIT_XCPT_BP:
5001	{
5002	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5003	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5004	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5005	}
5006
5007	case SVM_EXIT_READ_CR0:
5008	case SVM_EXIT_READ_CR3:
5009	case SVM_EXIT_READ_CR4:
5010	{
5011	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5012	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5013	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5014	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5015	}
5016
5017	case SVM_EXIT_CR0_SEL_WRITE:
5018	{
5019	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5020	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5021	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5022	}
5023
5024	case SVM_EXIT_WRITE_CR0:
5025	case SVM_EXIT_WRITE_CR3:
5026	case SVM_EXIT_WRITE_CR4:
5027	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5028	{
5029	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5030	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5031
5032	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5033	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5034	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5035	}
5036
5037	case SVM_EXIT_PAUSE:
5038	{
5039	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5040	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5041	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5042	}
5043
5044	case SVM_EXIT_VINTR:
5045	{
5046	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5047	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5048	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5049	}
5050
5051	case SVM_EXIT_INTR:
5052	case SVM_EXIT_NMI:
5053	case SVM_EXIT_SMI:
5054	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5055	{
5056	/*
5057	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5058	*
5059	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5060	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5061	* nested-guest #VMEXIT.
5062	*
5063	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5064	* while trying to inject interrupts, see comment at the top of this function.
5065	*/
5066	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5067	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5068	}
5069
5070	case SVM_EXIT_FERR_FREEZE:
5071	{
5072	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5073	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5074	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5075	}
5076
5077	case SVM_EXIT_INVLPG:
5078	{
5079	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5080	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5081	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5082	}
5083
5084	case SVM_EXIT_WBINVD:
5085	{
5086	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5087	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5088	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5089	}
5090
5091	case SVM_EXIT_INVD:
5092	{
5093	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5094	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5095	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5096	}
5097
5098	case SVM_EXIT_RDPMC:
5099	{
5100	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5101	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5102	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5103	}
5104
5105	default:
5106	{
5107	switch (uExitCode)
5108	{
5109	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5110	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5111	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5112	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5113	{
5114	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5115	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5116	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5117	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5118	}
5119
5120	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5121	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5122	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5123	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5124	{
5125	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5126	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5127	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5128	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5129	}
5130
5131	case SVM_EXIT_XCPT_DE:
5132	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5133	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5134	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5135	case SVM_EXIT_XCPT_OF:
5136	case SVM_EXIT_XCPT_BR:
5137	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5138	case SVM_EXIT_XCPT_NM:
5139	case SVM_EXIT_XCPT_DF:
5140	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5141	case SVM_EXIT_XCPT_TS:
5142	case SVM_EXIT_XCPT_NP:
5143	case SVM_EXIT_XCPT_SS:
5144	case SVM_EXIT_XCPT_GP:
5145	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5146	case SVM_EXIT_XCPT_15: /* Reserved. */
5147	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5148	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5149	case SVM_EXIT_XCPT_MC:
5150	case SVM_EXIT_XCPT_XF:
5151	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5152	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5153	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5154	{
5155	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5156	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5157	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5158	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5159	}
5160
5161	case SVM_EXIT_XSETBV:
5162	{
5163	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5164	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5165	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5166	}
5167
5168	case SVM_EXIT_TASK_SWITCH:
5169	{
5170	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5171	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5172	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5173	}
5174
5175	case SVM_EXIT_IRET:
5176	{
5177	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5178	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5179	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5180	}
5181
5182	case SVM_EXIT_SHUTDOWN:
5183	{
5184	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5185	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5186	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5187	}
5188
5189	case SVM_EXIT_VMMCALL:
5190	{
5191	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5192	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5193	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5194	}
5195
5196	case SVM_EXIT_CLGI:
5197	{
5198	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5199	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5200	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5201	}
5202
5203	case SVM_EXIT_STGI:
5204	{
5205	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5206	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5207	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5208	}
5209
5210	case SVM_EXIT_VMLOAD:
5211	{
5212	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5213	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5214	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5215	}
5216
5217	case SVM_EXIT_VMSAVE:
5218	{
5219	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5220	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5221	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5222	}
5223
5224	case SVM_EXIT_INVLPGA:
5225	{
5226	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx,