HMSVMR0.cpp

Last change on this file was 108996, checked in by vboxsync, 3 weeks ago
VMM: bugref:3409 Build fix.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 381.5 KB

Line
1	/* $Id: HMSVMR0.cpp 108996 2025-04-16 06:47:47Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_HM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/thread.h>
36
37	#include <VBox/vmm/pdmapi.h>
38	#include <VBox/vmm/dbgf.h>
39	#include <VBox/vmm/iem.h>
40	#include <VBox/vmm/iom.h>
41	#include <VBox/vmm/tm.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/gcm.h>
44	#include <VBox/vmm/gim.h>
45	#include <VBox/vmm/pdmapic.h>
46	#include "HMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include <VBox/err.h>
49	#include "HMSVMR0.h"
50	#include "dtrace/VBoxVMM.h"
51
52	#ifdef DEBUG_ramshankar
53	# define HMSVM_SYNC_FULL_GUEST_STATE
54	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
55	# define HMSVM_ALWAYS_TRAP_PF
56	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
57	#endif
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	#ifdef VBOX_WITH_STATISTICS
64	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
66	if ((u64ExitCode) == SVM_EXIT_NPF) \
67	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
68	else \
69	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
70	} while (0)
71
72	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
74	if ((u64ExitCode) == SVM_EXIT_NPF) \
75	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
76	else \
77	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
78	} while (0)
79
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
81	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
82	if ((u64ExitCode) == SVM_EXIT_NPF) \
83	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
84	else \
85	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
86	} while (0)
87	#else
88	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
89	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
90	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
91	#endif /* !VBOX_WITH_STATISTICS */
92
93	/** If we decide to use a function table approach this can be useful to
94	* switch to a "static DECLCALLBACK(int)". */
95	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
96
97	/**
98	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
99	* guest using hardware-assisted SVM.
100	*
101	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
102	* are swapped and restored across the world-switch and also registers like
103	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
104	* \#VMEXIT.
105	*/
106	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
107	\| CPUMCTX_EXTRN_RFLAGS \
108	\| CPUMCTX_EXTRN_RAX \
109	\| CPUMCTX_EXTRN_RSP \
110	\| CPUMCTX_EXTRN_SREG_MASK \
111	\| CPUMCTX_EXTRN_CR0 \
112	\| CPUMCTX_EXTRN_CR2 \
113	\| CPUMCTX_EXTRN_CR3 \
114	\| CPUMCTX_EXTRN_TABLE_MASK \
115	\| CPUMCTX_EXTRN_DR6 \
116	\| CPUMCTX_EXTRN_DR7 \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_HWVIRT \
121	\| CPUMCTX_EXTRN_INHIBIT_INT \
122	\| CPUMCTX_EXTRN_HM_SVM_MASK)
123
124	/**
125	* Subset of the guest-CPU state that is shared between the guest and host.
126	*/
127	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
128
129	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
130	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
131	do { \
132	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
133	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
134	} while (0)
135
136	/** Assert that the required state bits are fetched. */
137	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
151
152	/** Assert that we're not executing a nested-guest. */
153	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
154	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
155	#else
156	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
157	#endif
158
159	/** Assert that we're executing a nested-guest. */
160	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
161	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
162	#else
163	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
164	#endif
165
166	/** Macro for checking and returning from the using function for
167	* \#VMEXIT intercepts that maybe caused during delivering of another
168	* event in the guest. */
169	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
170	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
171	do \
172	{ \
173	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
174	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
175	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
176	else if ( rc == VINF_EM_RESET \
177	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
178	{ \
179	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
180	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
181	} \
182	else \
183	return rc; \
184	} while (0)
185	#else
186	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
187	do \
188	{ \
189	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
190	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
191	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
192	else \
193	return rc; \
194	} while (0)
195	#endif
196
197	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
198	* instruction that exited. */
199	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
200	do { \
201	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
202	(a_rc) = VINF_EM_DBG_STEPPED; \
203	} while (0)
204
205	/** Validate segment descriptor granularity bit. */
206	#ifdef VBOX_STRICT
207	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
208	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
209	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
210	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
211	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
212	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
213	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
214	#else
215	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
216	#endif
217
218	/**
219	* Exception bitmap mask for all contributory exceptions.
220	*
221	* Page fault is deliberately excluded here as it's conditional as to whether
222	* it's contributory or benign. Page faults are handled separately.
223	*/
224	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
225	\| RT_BIT(X86_XCPT_DE))
226
227	/**
228	* Mandatory/unconditional guest control intercepts.
229	*
230	* SMIs can and do happen in normal operation. We need not intercept them
231	* while executing the guest (or nested-guest).
232	*/
233	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
234	\| SVM_CTRL_INTERCEPT_NMI \
235	\| SVM_CTRL_INTERCEPT_INIT \
236	\| SVM_CTRL_INTERCEPT_RDPMC \
237	\| SVM_CTRL_INTERCEPT_CPUID \
238	\| SVM_CTRL_INTERCEPT_RSM \
239	\| SVM_CTRL_INTERCEPT_HLT \
240	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
241	\| SVM_CTRL_INTERCEPT_MSR_PROT \
242	\| SVM_CTRL_INTERCEPT_INVLPGA \
243	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
244	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
245	\| SVM_CTRL_INTERCEPT_VMRUN \
246	\| SVM_CTRL_INTERCEPT_SKINIT \
247	\| SVM_CTRL_INTERCEPT_WBINVD \
248	\| SVM_CTRL_INTERCEPT_MONITOR \
249	\| SVM_CTRL_INTERCEPT_MWAIT \
250	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
251	\| SVM_CTRL_INTERCEPT_XSETBV)
252
253	/** @name VMCB Clean Bits.
254	*
255	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
256	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
257	* memory.
258	*
259	* @{ */
260	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
261	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
262	/** I/O permission bitmap, MSR permission bitmap. */
263	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
264	/** ASID. */
265	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
266	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
267	V_INTR_VECTOR. */
268	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
269	/** Nested Paging: Nested CR3 (nCR3), PAT. */
270	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
271	/** Control registers (CR0, CR3, CR4, EFER). */
272	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
273	/** Debug registers (DR6, DR7). */
274	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
275	/** GDT, IDT limit and base. */
276	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
277	/** Segment register: CS, SS, DS, ES limit and base. */
278	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
279	/** CR2.*/
280	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
281	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
282	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
283	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
284	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
285	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
286	/** Mask of all valid VMCB Clean bits. */
287	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
288	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
289	\| HMSVM_VMCB_CLEAN_ASID \
290	\| HMSVM_VMCB_CLEAN_INT_CTRL \
291	\| HMSVM_VMCB_CLEAN_NP \
292	\| HMSVM_VMCB_CLEAN_CRX_EFER \
293	\| HMSVM_VMCB_CLEAN_DRX \
294	\| HMSVM_VMCB_CLEAN_DT \
295	\| HMSVM_VMCB_CLEAN_SEG \
296	\| HMSVM_VMCB_CLEAN_CR2 \
297	\| HMSVM_VMCB_CLEAN_LBR \
298	\| HMSVM_VMCB_CLEAN_AVIC)
299	/** @} */
300
301	/**
302	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
303	*/
304	typedef enum SVMMSREXITREAD
305	{
306	/** Reading this MSR causes a \#VMEXIT. */
307	SVMMSREXIT_INTERCEPT_READ = 0xb,
308	/** Reading this MSR does not cause a \#VMEXIT. */
309	SVMMSREXIT_PASSTHRU_READ
310	} SVMMSREXITREAD;
311
312	/**
313	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
314	*/
315	typedef enum SVMMSREXITWRITE
316	{
317	/** Writing to this MSR causes a \#VMEXIT. */
318	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a \#VMEXIT. */
320	SVMMSREXIT_PASSTHRU_WRITE
321	} SVMMSREXITWRITE;
322
323	/**
324	* SVM \#VMEXIT handler.
325	*
326	* @returns Strict VBox status code.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pSvmTransient Pointer to the SVM-transient structure.
329	*/
330	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
331
332
333	/*********************************************************************************************************************************
334	* Internal Functions *
335	*********************************************************************************************************************************/
336	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
337	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
338
339
340	/** @name \#VMEXIT handlers.
341	* @{
342	*/
343	static FNSVMEXITHANDLER hmR0SvmExitIntr;
344	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
345	static FNSVMEXITHANDLER hmR0SvmExitInvd;
346	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
347	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
348	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
349	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
350	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
351	static FNSVMEXITHANDLER hmR0SvmExitHlt;
352	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
353	static FNSVMEXITHANDLER hmR0SvmExitMwait;
354	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
355	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
356	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
357	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
358	static FNSVMEXITHANDLER hmR0SvmExitMsr;
359	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
360	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
361	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
362	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
363	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
364	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
365	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
366	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
367	static FNSVMEXITHANDLER hmR0SvmExitPause;
368	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
369	static FNSVMEXITHANDLER hmR0SvmExitIret;
370	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
371	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
372	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
373	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
374	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
375	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
376	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
377	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
378	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
379	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
380	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
381	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
382	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
383	static FNSVMEXITHANDLER hmR0SvmExitClgi;
384	static FNSVMEXITHANDLER hmR0SvmExitStgi;
385	static FNSVMEXITHANDLER hmR0SvmExitVmload;
386	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
387	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
388	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
389	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
390	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
391	#endif
392	/** @} */
393
394	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
395	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
396	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
397	#endif
398	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
399
400
401	/*********************************************************************************************************************************
402	* Global Variables *
403	*********************************************************************************************************************************/
404	/** Ring-0 memory object for the IO bitmap. */
405	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
406	/** Physical address of the IO bitmap. */
407	static RTHCPHYS g_HCPhysIOBitmap;
408	/** Pointer to the IO bitmap. */
409	static R0PTRTYPE(void *) g_pvIOBitmap;
410
411	#ifdef VBOX_STRICT
412	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
413	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
414	# define HMSVM_LOG_CS RT_BIT_32(2)
415	# define HMSVM_LOG_SS RT_BIT_32(3)
416	# define HMSVM_LOG_FS RT_BIT_32(4)
417	# define HMSVM_LOG_GS RT_BIT_32(5)
418	# define HMSVM_LOG_LBR RT_BIT_32(6)
419	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
420	\| HMSVM_LOG_CR_REGS \
421	\| HMSVM_LOG_CS \
422	\| HMSVM_LOG_SS \
423	\| HMSVM_LOG_FS \
424	\| HMSVM_LOG_GS \
425	\| HMSVM_LOG_LBR)
426
427	/**
428	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
429	*
430	* @param pVCpu The cross context virtual CPU structure.
431	* @param pVmcb Pointer to the VM control block.
432	* @param pszPrefix Log prefix.
433	* @param fFlags Log flags, see HMSVM_LOG_XXX.
434	* @param uVerbose The verbosity level, currently unused.
435	*/
436	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
437	{
438	RT_NOREF2(pVCpu, uVerbose);
439	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
440
441	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
442	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
443
444	if (fFlags & HMSVM_LOG_RBP_RSP)
445	{
446	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
447	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
448	}
449
450	if (fFlags & HMSVM_LOG_CR_REGS)
451	{
452	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
453	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
454	}
455
456	if (fFlags & HMSVM_LOG_CS)
457	{
458	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
459	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
460	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
461	}
462	if (fFlags & HMSVM_LOG_SS)
463	{
464	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
465	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
466	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
467	}
468	if (fFlags & HMSVM_LOG_FS)
469	{
470	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
471	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
472	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
473	}
474	if (fFlags & HMSVM_LOG_GS)
475	{
476	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
477	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
478	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
479	}
480
481	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
482	if (fFlags & HMSVM_LOG_LBR)
483	{
484	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
485	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
486	}
487	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
488	}
489	#endif /* VBOX_STRICT */
490
491
492	/**
493	* Sets up and activates AMD-V on the current CPU.
494	*
495	* @returns VBox status code.
496	* @param pHostCpu The HM physical-CPU structure.
497	* @param pVM The cross context VM structure. Can be
498	* NULL after a resume!
499	* @param pvCpuPage Pointer to the global CPU page.
500	* @param HCPhysCpuPage Physical address of the global CPU page.
501	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
502	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
503	* unused).
504	*/
505	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
506	PCSUPHWVIRTMSRS pHwvirtMsrs)
507	{
508	Assert(!fEnabledByHost);
509	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
510	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
511	Assert(pvCpuPage); NOREF(pvCpuPage);
512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
513
514	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
515
516	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
517	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
518
519	/*
520	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
521	*/
522	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
523	if (u64HostEfer & MSR_K6_EFER_SVME)
524	{
525	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
526	if ( pVM
527	&& pVM->hm.s.svm.fIgnoreInUseError)
528	pHostCpu->fIgnoreAMDVInUseError = true;
529
530	if (!pHostCpu->fIgnoreAMDVInUseError)
531	{
532	ASMSetFlags(fEFlags);
533	return VERR_SVM_IN_USE;
534	}
535	}
536
537	/* Turn on AMD-V in the EFER MSR. */
538	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
539
540	/* Write the physical page address where the CPU will store the host state while executing the VM. */
541	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
542
543	/* Restore interrupts. */
544	ASMSetFlags(fEFlags);
545
546	/*
547	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
548	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
549	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
550	* entirely with before executing any guest code.
551	*/
552	pHostCpu->fFlushAsidBeforeUse = true;
553
554	/*
555	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
556	*/
557	++pHostCpu->cTlbFlushes;
558
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Deactivates AMD-V on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pHostCpu The HM physical-CPU structure.
568	* @param pvCpuPage Pointer to the global CPU page.
569	* @param HCPhysCpuPage Physical address of the global CPU page.
570	*/
571	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
572	{
573	RT_NOREF1(pHostCpu);
574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
575	AssertReturn( HCPhysCpuPage
576	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
577	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
578
579	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
580	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
581
582	/* Turn off AMD-V in the EFER MSR. */
583	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
584	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
585
586	/* Invalidate host state physical address. */
587	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
588
589	/* Restore interrupts. */
590	ASMSetFlags(fEFlags);
591
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Does global AMD-V initialization (called during module initialization).
598	*
599	* @returns VBox status code.
600	*/
601	VMMR0DECL(int) SVMR0GlobalInit(void)
602	{
603	/*
604	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
605	* intercept all IO accesses, it's done once globally here instead of per-VM.
606	*/
607	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
608	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT,
609	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
610	if (RT_FAILURE(rc))
611	return rc;
612
613	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
614	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
615
616	/* Set all bits to intercept all IO accesses. */
617	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
618
619	return VINF_SUCCESS;
620	}
621
622
623	/**
624	* Does global AMD-V termination (called during module termination).
625	*/
626	VMMR0DECL(void) SVMR0GlobalTerm(void)
627	{
628	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
629	{
630	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
631	g_pvIOBitmap = NULL;
632	g_HCPhysIOBitmap = 0;
633	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
634	}
635	}
636
637
638	/**
639	* Frees any allocated per-VCPU structures for a VM.
640	*
641	* @param pVM The cross context VM structure.
642	*/
643	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
644	{
645	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
646	{
647	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
648	AssertPtr(pVCpu);
649
650	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
651	{
652	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
653	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
654	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
655	}
656
657	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
658	{
659	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
660	pVCpu->hmr0.s.svm.pVmcb = NULL;
661	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
662	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
663	}
664
665	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
666	{
667	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
668	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
669	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
670	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
671	}
672	}
673	}
674
675
676	/**
677	* Sets pfnVMRun to the best suited variant.
678	*
679	* This must be called whenever anything changes relative to the SVMR0VMRun
680	* variant selection:
681	* - pVCpu->hm.s.fLoadSaveGuestXcr0
682	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
683	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
684	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
685	* - Perhaps: CPUMCTX.fXStateMask (windows only)
686	*
687	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
688	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
689	*/
690	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
691	{
692	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
693	{
694	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit_SansSpecCtrl },
695	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit_SansSpecCtrl },
696	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit_SansSpecCtrl },
697	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit_SansSpecCtrl },
698	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit_SansSpecCtrl },
699	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit_SansSpecCtrl },
700	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit_SansSpecCtrl },
701	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit_SansSpecCtrl },
702	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit_WithSpecCtrl },
703	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit_WithSpecCtrl },
704	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit_WithSpecCtrl },
705	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit_WithSpecCtrl },
706	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit_WithSpecCtrl },
707	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit_WithSpecCtrl },
708	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit_WithSpecCtrl },
709	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit_WithSpecCtrl },
710	};
711	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
712	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
713	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0)
714	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_SPEC_CTRL ? 8 : 0);
715	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
716	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
717	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
718	}
719
720
721	/**
722	* Selector FNHMSVMVMRUN implementation.
723	*/
724	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
725	{
726	hmR0SvmUpdateVmRunFunction(pVCpu);
727	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
728	}
729
730
731	/**
732	* Does per-VM AMD-V initialization.
733	*
734	* @returns VBox status code.
735	* @param pVM The cross context VM structure.
736	*/
737	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
738	{
739	int rc = VERR_INTERNAL_ERROR_5;
740
741	/*
742	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
743	*/
744	uint32_t u32Family;
745	uint32_t u32Model;
746	uint32_t u32Stepping;
747	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
748	{
749	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
750	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
751	}
752
753	/*
754	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
755	*/
756	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
757	{
758	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
759	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
760	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
761	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
762	}
763
764	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
765	{
766	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
767
768	/*
769	* Initialize the hardware-assisted SVM guest-execution handler.
770	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
771	*/
772	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
773
774	/*
775	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
776	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
777	*/
778	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
779	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
780	if (RT_FAILURE(rc))
781	goto failure_cleanup;
782
783	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
784	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
785	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
786
787	/*
788	* Allocate one page for the guest-state VMCB.
789	*/
790	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
791	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
792	if (RT_FAILURE(rc))
793	goto failure_cleanup;
794
795	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
796	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
797	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
798
799	/*
800	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
801	* SVM to not require one.
802	*/
803	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
804	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
805	if (RT_FAILURE(rc))
806	goto failure_cleanup;
807
808	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
809	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
810	/* Set all bits to intercept all MSR accesses (changed later on). */
811	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
812	}
813
814	return VINF_SUCCESS;
815
816	failure_cleanup:
817	hmR0SvmFreeStructs(pVM);
818	return rc;
819	}
820
821
822	/**
823	* Does per-VM AMD-V termination.
824	*
825	* @returns VBox status code.
826	* @param pVM The cross context VM structure.
827	*/
828	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
829	{
830	hmR0SvmFreeStructs(pVM);
831	return VINF_SUCCESS;
832	}
833
834
835	/**
836	* Returns whether the VMCB Clean Bits feature is supported.
837	*
838	* @returns @c true if supported, @c false otherwise.
839	* @param pVCpu The cross context virtual CPU structure.
840	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
841	*/
842	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
843	{
844	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
845	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
846	if (!fIsNestedGuest)
847	return fHostVmcbCleanBits;
848	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
849	}
850
851
852	/**
853	* Returns whether the decode assists feature is supported.
854	*
855	* @returns @c true if supported, @c false otherwise.
856	* @param pVCpu The cross context virtual CPU structure.
857	*/
858	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
859	{
860	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
861	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
862	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
863	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
864	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
865	#endif
866	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
867	}
868
869
870	/**
871	* Returns whether the NRIP_SAVE feature is supported.
872	*
873	* @returns @c true if supported, @c false otherwise.
874	* @param pVCpu The cross context virtual CPU structure.
875	*/
876	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
877	{
878	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
879	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
880	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
881	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
882	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
883	#endif
884	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
885	}
886
887
888	/**
889	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
890	*
891	* @param pVCpu The cross context virtual CPU structure.
892	* @param pbMsrBitmap Pointer to the MSR bitmap.
893	* @param idMsr The MSR for which the permissions are being set.
894	* @param enmRead MSR read permissions.
895	* @param enmWrite MSR write permissions.
896	*
897	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
898	* caller needs to take care of this.
899	*/
900	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
901	SVMMSREXITWRITE enmWrite)
902	{
903	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
904	uint16_t offMsrpm;
905	uint8_t uMsrpmBit;
906	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
907	AssertRC(rc);
908
909	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
910	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
911
912	pbMsrBitmap += offMsrpm;
913	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
914	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
915	else
916	{
917	if (!fInNestedGuestMode)
918	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
919	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
920	else
921	{
922	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
923	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
924	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
925	else
926	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
927	}
928	#endif
929	}
930
931	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
932	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
933	else
934	{
935	if (!fInNestedGuestMode)
936	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
937	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
938	else
939	{
940	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
941	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
942	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
943	else
944	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
945	}
946	#endif
947	}
948	}
949
950
951	/**
952	* Sets up AMD-V for the specified VM.
953	* This function is only called once per-VM during initalization.
954	*
955	* @returns VBox status code.
956	* @param pVM The cross context VM structure.
957	*/
958	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
959	{
960	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
961	AssertReturn(pVM, VERR_INVALID_PARAMETER);
962
963	/*
964	* Validate and copy over some parameters.
965	*/
966	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
967	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
968	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
969	pVM->hmr0.s.fNestedPaging = fNestedPaging;
970	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
971
972	/*
973	* Determin some configuration parameters.
974	*/
975	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
976	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
977	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
978
979	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
980	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
981
982	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
983	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
984	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
985
986	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
987	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
988	#endif
989
990	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
991	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
992	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
993	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
994
995	/* Always trap #AC for reasons of security. */
996	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
997
998	/* Always trap #DB for reasons of security. */
999	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
1000
1001	/* Trap exceptions unconditionally (debug purposes). */
1002	#ifdef HMSVM_ALWAYS_TRAP_PF
1003	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
1004	#endif
1005	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1006	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
1007	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
1008	\| RT_BIT_32(X86_XCPT_DE)
1009	\| RT_BIT_32(X86_XCPT_NM)
1010	\| RT_BIT_32(X86_XCPT_UD)
1011	\| RT_BIT_32(X86_XCPT_NP)
1012	\| RT_BIT_32(X86_XCPT_SS)
1013	\| RT_BIT_32(X86_XCPT_GP)
1014	\| RT_BIT_32(X86_XCPT_PF)
1015	\| RT_BIT_32(X86_XCPT_MF)
1016	;
1017	#endif
1018
1019	/* Apply the exceptions intercepts needed by the GIM provider. */
1020	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1021	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1022
1023	/* Apply the exceptions intercepts needed by the GCM fixers. */
1024	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1025	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1026
1027	/* The mesa 3d driver hack needs #GP. */
1028	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1029	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1030
1031	/* Set up unconditional intercepts and conditions. */
1032	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1033	\| SVM_CTRL_INTERCEPT_VMMCALL
1034	\| SVM_CTRL_INTERCEPT_VMSAVE
1035	\| SVM_CTRL_INTERCEPT_VMLOAD
1036	\| SVM_CTRL_INTERCEPT_CLGI
1037	\| SVM_CTRL_INTERCEPT_STGI;
1038
1039	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1040	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1041	#endif
1042
1043	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1044	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1045	{
1046	/* Virtualized VMSAVE/VMLOAD. */
1047	if (fUseVirtVmsaveVmload)
1048	{
1049	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1050	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1051	\| SVM_CTRL_INTERCEPT_VMLOAD);
1052	}
1053	else
1054	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1055
1056	/* Virtual GIF. */
1057	if (fUseVGif)
1058	{
1059	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1060	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1061	\| SVM_CTRL_INTERCEPT_STGI);
1062	}
1063	else
1064	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1065	}
1066	else
1067	#endif
1068	{
1069	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1070	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1071	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1072	}
1073
1074	/* CR4 writes must always be intercepted for tracking PGM mode changes and
1075	AVX (for XCR0 syncing during worlds switching). */
1076	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1077
1078	/* Intercept all DRx reads and writes by default. Changed later on. */
1079	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1080	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1081
1082	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1083	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1084
1085	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1086	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1087	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1088
1089	/* Set the IO permission bitmap physical addresses. */
1090	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1091
1092	/* LBR virtualization. */
1093	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1094
1095	/* The host ASID MBZ, for the guest start with 1. */
1096	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1097
1098	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1099	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1100
1101	/* Without Nested Paging, we need additionally intercepts. */
1102	if (!fNestedPaging)
1103	{
1104	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1105	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1106	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1107
1108	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1109	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1110	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1111
1112	/* Page faults must be intercepted to implement shadow paging. */
1113	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1114	}
1115
1116	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1117	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1118	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1119
1120	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1121	if (fUsePauseFilter)
1122	{
1123	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1124	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1125	if (fPauseFilterThreshold)
1126	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1127	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1128	}
1129
1130	/*
1131	* Setup the MSR permission bitmap.
1132	* The following MSRs are saved/restored automatically during the world-switch.
1133	* Don't intercept guest read/write accesses to these MSRs.
1134	*/
1135	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1136	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1137	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1138	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1139	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1140	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1141	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1142	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1143	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1144	{
1145	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1146	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1147	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1148	}
1149	else
1150	{
1151	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1152	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1153	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1154	}
1155	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1156
1157	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1158	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1159
1160	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1161	{
1162	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1163	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1164	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1165	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1166
1167	/* Copy the VMCB control area. */
1168	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1169
1170	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1171	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1172	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1173	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1174
1175	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1176	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1177
1178	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1179	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1180	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1181	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1182	}
1183
1184	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1185	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1186	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1187	#else
1188	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1189	#endif
1190	return VINF_SUCCESS;
1191	}
1192
1193
1194	/**
1195	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1196	*
1197	* @returns Pointer to the current context VMCB.
1198	* @param pVCpu The cross context virtual CPU structure.
1199	*/
1200	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1201	{
1202	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1203	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1204	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1205	#endif
1206	return pVCpu->hmr0.s.svm.pVmcb;
1207	}
1208
1209
1210	/**
1211	* Gets a pointer to the nested-guest VMCB cache.
1212	*
1213	* @returns Pointer to the nested-guest VMCB cache.
1214	* @param pVCpu The cross context virtual CPU structure.
1215	*/
1216	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1217	{
1218	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1219	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1220	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1221	#else
1222	RT_NOREF(pVCpu);
1223	return NULL;
1224	#endif
1225	}
1226
1227
1228	/**
1229	* Invalidates a guest page by guest virtual address.
1230	*
1231	* @returns VBox status code.
1232	* @param pVCpu The cross context virtual CPU structure.
1233	* @param GCVirt Guest virtual address of the page to invalidate.
1234	*/
1235	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1236	{
1237	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1238
1239	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1240
1241	/* Skip it if a TLB flush is already pending. */
1242	if (!fFlushPending)
1243	{
1244	Log4Func(("%#RGv\n", GCVirt));
1245
1246	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1247	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1248
1249	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1250	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1251	}
1252	return VINF_SUCCESS;
1253	}
1254
1255
1256	/**
1257	* Flushes the appropriate tagged-TLB entries.
1258	*
1259	* @param pHostCpu The HM physical-CPU structure.
1260	* @param pVCpu The cross context virtual CPU structure.
1261	* @param pVmcb Pointer to the VM control block.
1262	*/
1263	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1264	{
1265	/*
1266	* Force a TLB flush for the first world switch if the current CPU differs from the one
1267	* we ran on last. This can happen both for start & resume due to long jumps back to
1268	* ring-3.
1269	*
1270	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1271	* correlation between it and the physical CPU.
1272	*
1273	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1274	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1275	*/
1276	bool fNewAsid = false;
1277	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1278	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1279	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1280	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1281	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1282	#endif
1283	)
1284	{
1285	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1286	pVCpu->hmr0.s.fForceTLBFlush = true;
1287	fNewAsid = true;
1288	}
1289
1290	/* Set TLB flush state as checked until we return from the world switch. */
1291	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1292
1293	/* Check for explicit TLB flushes. */
1294	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1295	{
1296	pVCpu->hmr0.s.fForceTLBFlush = true;
1297	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1298	}
1299
1300	/*
1301	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1302	* This Host CPU requirement takes precedence.
1303	*/
1304	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1305	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1306	{
1307	pHostCpu->uCurrentAsid = 1;
1308	pVCpu->hmr0.s.uCurrentAsid = 1;
1309	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1310	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1311	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1312
1313	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1314	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1315	}
1316	else
1317	{
1318	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1319	if (pVCpu->hmr0.s.fForceTLBFlush)
1320	{
1321	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1322	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1323
1324	if (fNewAsid)
1325	{
1326	++pHostCpu->uCurrentAsid;
1327
1328	bool fHitASIDLimit = false;
1329	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1330	{
1331	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1332	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1333	fHitASIDLimit = true;
1334	}
1335
1336	if ( fHitASIDLimit
1337	\|\| pHostCpu->fFlushAsidBeforeUse)
1338	{
1339	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1340	pHostCpu->fFlushAsidBeforeUse = false;
1341	}
1342
1343	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1344	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1345	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1346	}
1347	else
1348	{
1349	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1350	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1351	else
1352	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1353	}
1354
1355	pVCpu->hmr0.s.fForceTLBFlush = false;
1356	}
1357	}
1358
1359	/* Update VMCB with the ASID. */
1360	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1361	{
1362	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1363	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1364	}
1365
1366	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1367	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1368	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1369	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1370	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1371	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1372	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1373	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1374
1375	#ifdef VBOX_WITH_STATISTICS
1376	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1377	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1378	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1379	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1380	{
1381	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1382	}
1383	else
1384	{
1385	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1386	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1387	}
1388	#endif
1389	}
1390
1391
1392	/**
1393	* Sets an exception intercept in the specified VMCB.
1394	*
1395	* @param pVmcb Pointer to the VM control block.
1396	* @param uXcpt The exception (X86_XCPT_*).
1397	*/
1398	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1399	{
1400	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1401	{
1402	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1403	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1404	}
1405	}
1406
1407
1408	/**
1409	* Clears an exception intercept in the specified VMCB.
1410	*
1411	* @param pVCpu The cross context virtual CPU structure.
1412	* @param pVmcb Pointer to the VM control block.
1413	* @param uXcpt The exception (X86_XCPT_*).
1414	*
1415	* @remarks This takes into account if we're executing a nested-guest and only
1416	* removes the exception intercept if both the guest -and- nested-guest
1417	* are not intercepting it.
1418	*/
1419	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1420	{
1421	Assert(uXcpt != X86_XCPT_DB);
1422	Assert(uXcpt != X86_XCPT_AC);
1423	Assert(uXcpt != X86_XCPT_GP);
1424	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1425	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1426	{
1427	bool fRemove = true;
1428	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1429	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1430	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1431	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1432	{
1433	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1434	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1435	}
1436	# else
1437	RT_NOREF(pVCpu);
1438	# endif
1439	if (fRemove)
1440	{
1441	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1442	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1443	}
1444	}
1445	#else
1446	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1447	#endif
1448	}
1449
1450
1451	/**
1452	* Sets a control intercept in the specified VMCB.
1453	*
1454	* @param pVmcb Pointer to the VM control block.
1455	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1456	*/
1457	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1458	{
1459	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1460	{
1461	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1462	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1463	}
1464	}
1465
1466
1467	/**
1468	* Clears a control intercept in the specified VMCB.
1469	*
1470	* @returns @c true if the intercept is still set, @c false otherwise.
1471	* @param pVCpu The cross context virtual CPU structure.
1472	* @param pVmcb Pointer to the VM control block.
1473	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1474	*
1475	* @remarks This takes into account if we're executing a nested-guest and only
1476	* removes the control intercept if both the guest -and- nested-guest
1477	* are not intercepting it.
1478	*/
1479	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1480	{
1481	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1482	{
1483	bool fRemove = true;
1484	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1485	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1486	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1487	{
1488	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1489	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1490	}
1491	#else
1492	RT_NOREF(pVCpu);
1493	#endif
1494	if (fRemove)
1495	{
1496	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1497	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1498	}
1499	}
1500
1501	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1502	}
1503
1504
1505	/**
1506	* Exports the guest (or nested-guest) CR0 into the VMCB.
1507	*
1508	* @param pVCpu The cross context virtual CPU structure.
1509	* @param pVmcb Pointer to the VM control block.
1510	*
1511	* @remarks This assumes we always pre-load the guest FPU.
1512	* @remarks No-long-jump zone!!!
1513	*/
1514	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1515	{
1516	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1517
1518	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1519	uint64_t const uGuestCr0 = pCtx->cr0;
1520	uint64_t uShadowCr0 = uGuestCr0;
1521
1522	/* Always enable caching. */
1523	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1524
1525	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1526	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1527	{
1528	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1529	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1530	}
1531
1532	/*
1533	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1534	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1535	* see @bugref{7243#c103}.
1536	*/
1537	if (!(uGuestCr0 & X86_CR0_NE))
1538	{
1539	uShadowCr0 \|= X86_CR0_NE;
1540	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1541	}
1542	else
1543	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1544
1545	/*
1546	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1547	*
1548	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1549	* see @bugref{6944}.
1550	*
1551	* We also don't ever want to honor weird things like cache disable from the guest.
1552	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1553	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1554	*/
1555	if (uShadowCr0 == uGuestCr0)
1556	{
1557	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1558	{
1559	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1560	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1561	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1562	}
1563	else
1564	{
1565	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1566	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1567	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1568	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1569	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1570	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1571	}
1572	}
1573	else
1574	{
1575	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1576	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1577	}
1578	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1579
1580	Assert(!RT_HI_U32(uShadowCr0));
1581	if (pVmcb->guest.u64CR0 != uShadowCr0)
1582	{
1583	pVmcb->guest.u64CR0 = uShadowCr0;
1584	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1585	}
1586	}
1587
1588
1589	/**
1590	* Exports the guest (or nested-guest) CR3 into the VMCB.
1591	*
1592	* @param pVCpu The cross context virtual CPU structure.
1593	* @param pVmcb Pointer to the VM control block.
1594	*
1595	* @remarks No-long-jump zone!!!
1596	*/
1597	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1598	{
1599	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1600
1601	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1602	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1603	if (pVM->hmr0.s.fNestedPaging)
1604	{
1605	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1606	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1607	pVmcb->guest.u64CR3 = pCtx->cr3;
1608	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1609	}
1610	else
1611	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1612
1613	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1614	}
1615
1616
1617	/**
1618	* Exports the guest (or nested-guest) CR4 into the VMCB.
1619	*
1620	* @param pVCpu The cross context virtual CPU structure.
1621	* @param pVmcb Pointer to the VM control block.
1622	*
1623	* @remarks No-long-jump zone!!!
1624	*/
1625	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1626	{
1627	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1628
1629	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1630	uint64_t uShadowCr4 = pCtx->cr4;
1631	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1632	{
1633	switch (pVCpu->hm.s.enmShadowMode)
1634	{
1635	case PGMMODE_REAL:
1636	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1637	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1638
1639	case PGMMODE_32_BIT: /* 32-bit paging. */
1640	uShadowCr4 &= ~X86_CR4_PAE;
1641	break;
1642
1643	case PGMMODE_PAE: /* PAE paging. */
1644	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1645	/** Must use PAE paging as we could use physical memory > 4 GB */
1646	uShadowCr4 \|= X86_CR4_PAE;
1647	break;
1648
1649	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1650	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1651	#ifdef VBOX_WITH_64_BITS_GUESTS
1652	break;
1653	#else
1654	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1655	#endif
1656
1657	default: /* shut up gcc */
1658	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1659	}
1660	}
1661
1662	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1663	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1664	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1665	{
1666	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1667	hmR0SvmUpdateVmRunFunction(pVCpu);
1668	}
1669
1670	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1671	if (uShadowCr4 == pCtx->cr4)
1672	{
1673	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1674	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1675	else
1676	{
1677	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1678	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1679	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1680	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1681	}
1682	}
1683	else
1684	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1685
1686	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking
1687	PGM mode changes and AVX (for XCR0 syncing during worlds switching). */
1688	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1689
1690	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1691	Assert(!RT_HI_U32(uShadowCr4));
1692	pVmcb->guest.u64CR4 = uShadowCr4;
1693	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1694
1695	return VINF_SUCCESS;
1696	}
1697
1698
1699	/**
1700	* Exports the guest (or nested-guest) control registers into the VMCB.
1701	*
1702	* @returns VBox status code.
1703	* @param pVCpu The cross context virtual CPU structure.
1704	* @param pVmcb Pointer to the VM control block.
1705	*
1706	* @remarks No-long-jump zone!!!
1707	*/
1708	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1709	{
1710	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1713	{
1714	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1715	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1716
1717	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1718	{
1719	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1720	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1721	}
1722
1723	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1724	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1725
1726	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1727	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1728	{
1729	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1730	if (RT_FAILURE(rc))
1731	return rc;
1732	}
1733
1734	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1735	}
1736	return VINF_SUCCESS;
1737	}
1738
1739
1740	/**
1741	* Exports the guest (or nested-guest) segment registers into the VMCB.
1742	*
1743	* @param pVCpu The cross context virtual CPU structure.
1744	* @param pVmcb Pointer to the VM control block.
1745	*
1746	* @remarks No-long-jump zone!!!
1747	*/
1748	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1749	{
1750	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1751	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1752
1753	/* Guest segment registers. */
1754	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1755	{
1756	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1757	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1758
1759	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1760	{
1761	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1762	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1763	}
1764
1765	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1766	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1767
1768	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1769	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1770
1771	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1772	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1773
1774	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1775	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1776
1777	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1778	}
1779
1780	/* Guest TR. */
1781	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1782	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1783
1784	/* Guest LDTR. */
1785	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1786	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1787
1788	/* Guest GDTR. */
1789	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1790	{
1791	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1792	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1793	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1794	}
1795
1796	/* Guest IDTR. */
1797	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1798	{
1799	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1800	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1801	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1802	}
1803
1804	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1805	\| HM_CHANGED_GUEST_TABLE_MASK);
1806	}
1807
1808
1809	/**
1810	* Exports the guest (or nested-guest) MSRs into the VMCB.
1811	*
1812	* @param pVCpu The cross context virtual CPU structure.
1813	* @param pVmcb Pointer to the VM control block.
1814	*
1815	* @remarks No-long-jump zone!!!
1816	*/
1817	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1818	{
1819	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1820	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1821
1822	/* Guest Sysenter MSRs. */
1823	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1824	{
1825	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1826	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1827
1828	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1829	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1830
1831	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1832	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1833	}
1834
1835	/*
1836	* Guest EFER MSR.
1837	* AMD-V requires guest EFER.SVME to be set. Weird.
1838	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1839	*/
1840	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1841	{
1842	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1843	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1844	}
1845
1846	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1847	if ( !CPUMIsGuestInLongModeEx(pCtx)
1848	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1849	{
1850	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1851	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1852	}
1853
1854	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1855	{
1856	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1857	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1858	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1859	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1860	}
1861
1862	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1863	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1864
1865	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1866	\| HM_CHANGED_GUEST_EFER_MSR
1867	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1868	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1869
1870	/*
1871	* Setup the PAT MSR (applicable for Nested Paging only).
1872	*
1873	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1874	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1875	*
1876	* While guests can modify and see the modified values through the shadow values,
1877	* we shall not honor any guest modifications of this MSR to ensure caching is always
1878	* enabled similar to how we clear CR0.CD and NW bits.
1879	*
1880	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1881	*/
1882	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1883
1884	/* Enable the last branch record bit if LBR virtualization is enabled. */
1885	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1886	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1887	}
1888
1889
1890	/**
1891	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1892	* the necessary intercepts accordingly.
1893	*
1894	* @param pVCpu The cross context virtual CPU structure.
1895	* @param pVmcb Pointer to the VM control block.
1896	*
1897	* @remarks No-long-jump zone!!!
1898	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1899	*/
1900	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1901	{
1902	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1903
1904	/** @todo Figure out stepping with nested-guest. */
1905	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1906	{
1907	/*
1908	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1909	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1910	* Instead, they are strictly only requested when the nested hypervisor intercepts
1911	* them -- handled while merging VMCB controls.
1912	*
1913	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1914	* then the nested-guest debug state should be actively loaded on the host so that
1915	* nested-guest reads/writes its own debug registers without causing VM-exits.
1916	*/
1917	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1918	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1919	&& !CPUMIsGuestDebugStateActive(pVCpu))
1920	{
1921	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1922	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1923	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1924	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1925	}
1926
1927	pVmcb->guest.u64DR6 = pCtx->dr[6];
1928	pVmcb->guest.u64DR7 = pCtx->dr[7];
1929	return;
1930	}
1931
1932	/*
1933	* Anyone single stepping on the host side? If so, we'll have to use the
1934	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1935	* the VMM level like the VT-x implementations does.
1936	*/
1937	bool fInterceptMovDRx = false;
1938	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1939	if (fStepping)
1940	{
1941	pVCpu->hmr0.s.fClearTrapFlag = true;
1942	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1943	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1944	}
1945
1946	if ( fStepping
1947	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1948	{
1949	/*
1950	* Use the combined guest and host DRx values found in the hypervisor
1951	* register set because the debugger has breakpoints active or someone
1952	* is single stepping on the host side.
1953	*
1954	* Note! DBGF expects a clean DR6 state before executing guest code.
1955	*/
1956	if (!CPUMIsHyperDebugStateActive(pVCpu))
1957	{
1958	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1959	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1960	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1961	}
1962
1963	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1964	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1965	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1966	{
1967	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1968	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1969	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1970	}
1971
1972	/** @todo If we cared, we could optimize to allow the guest to read registers
1973	* with the same values. */
1974	fInterceptMovDRx = true;
1975	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1976	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1977	}
1978	else
1979	{
1980	/*
1981	* Update DR6, DR7 with the guest values if necessary.
1982	*/
1983	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1984	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1985	{
1986	pVmcb->guest.u64DR7 = pCtx->dr[7];
1987	pVmcb->guest.u64DR6 = pCtx->dr[6];
1988	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1989	}
1990	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1991
1992	/*
1993	* If the guest has enabled debug registers, we need to load them prior to
1994	* executing guest code so they'll trigger at the right time.
1995	*/
1996	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1997	{
1998	if (!CPUMIsGuestDebugStateActive(pVCpu))
1999	{
2000	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
2001	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2002	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2003	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2004	}
2005	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2006	}
2007	/*
2008	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2009	* intercept #DB as DR6 is updated in the VMCB.
2010	*
2011	* Note! If we cared and dared, we could skip intercepting \#DB here.
2012	* However, \#DB shouldn't be performance critical, so we'll play safe
2013	* and keep the code similar to the VT-x code and always intercept it.
2014	*/
2015	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2016	fInterceptMovDRx = true;
2017	}
2018
2019	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2020	if (fInterceptMovDRx)
2021	{
2022	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2023	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2024	{
2025	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2026	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2027	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2028	}
2029	}
2030	else
2031	{
2032	if ( pVmcb->ctrl.u16InterceptRdDRx
2033	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2034	{
2035	pVmcb->ctrl.u16InterceptRdDRx = 0;
2036	pVmcb->ctrl.u16InterceptWrDRx = 0;
2037	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2038	}
2039	}
2040	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2041	}
2042
2043	/**
2044	* Exports the hardware virtualization state into the nested-guest
2045	* VMCB.
2046	*
2047	* @param pVCpu The cross context virtual CPU structure.
2048	* @param pVmcb Pointer to the VM control block.
2049	*
2050	* @remarks No-long-jump zone!!!
2051	*/
2052	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2053	{
2054	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2055
2056	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2057	{
2058	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2059	{
2060	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2061	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2062
2063	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2064	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2065	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2066	NOREF(pVM);
2067
2068	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2069	}
2070
2071	/*
2072	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2073	* since SVM doesn't have a preemption timer.
2074	*
2075	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2076	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2077	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2078	*/
2079	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2080	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2081	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2082	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2083	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2084	{
2085	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2086	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2087	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2088	}
2089	else
2090	{
2091	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2092	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2093	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2094	}
2095	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2096
2097	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2098	}
2099	}
2100
2101
2102	/**
2103	* Exports the guest APIC TPR state into the VMCB.
2104	*
2105	* @returns VBox status code.
2106	* @param pVCpu The cross context virtual CPU structure.
2107	* @param pVmcb Pointer to the VM control block.
2108	*/
2109	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2110	{
2111	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2112
2113	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2114	{
2115	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2116	if ( PDMHasApic(pVM)
2117	&& PDMApicIsEnabled(pVCpu))
2118	{
2119	bool fPendingIntr;
2120	uint8_t u8Tpr;
2121	int rc = PDMApicGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2122	AssertRCReturn(rc, rc);
2123
2124	/* Assume that we need to trap all TPR accesses and thus need not check on
2125	every #VMEXIT if we should update the TPR. */
2126	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2127	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2128
2129	if (!pVM->hm.s.fTprPatchingActive)
2130	{
2131	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2132	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2133
2134	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2135	can deliver the interrupt to the guest. */
2136	if (fPendingIntr)
2137	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2138	else
2139	{
2140	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2141	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2142	}
2143
2144	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2145	}
2146	else
2147	{
2148	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2149	pVmcb->guest.u64LSTAR = u8Tpr;
2150	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2151
2152	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2153	if (fPendingIntr)
2154	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2155	else
2156	{
2157	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2158	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2159	}
2160	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2161	}
2162	}
2163	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2164	}
2165	return VINF_SUCCESS;
2166	}
2167
2168
2169	/**
2170	* Sets up the exception interrupts required for guest execution in the VMCB.
2171	*
2172	* @param pVCpu The cross context virtual CPU structure.
2173	* @param pVmcb Pointer to the VM control block.
2174	*
2175	* @remarks No-long-jump zone!!!
2176	*/
2177	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2178	{
2179	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2180
2181	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2182	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2183	{
2184	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2185	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2186	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2187	else
2188	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2189
2190	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2191	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledSwBreakpoints)
2192	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2193	else
2194	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2195
2196	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2197	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2198	}
2199	}
2200
2201
2202	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2203	/**
2204	* Merges guest and nested-guest intercepts for executing the nested-guest using
2205	* hardware-assisted SVM.
2206	*
2207	* This merges the guest and nested-guest intercepts in a way that if the outer
2208	* guest intercept is set we need to intercept it in the nested-guest as
2209	* well.
2210	*
2211	* @param pVCpu The cross context virtual CPU structure.
2212	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2213	*/
2214	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2215	{
2216	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2217	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2218	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2219	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2220
2221	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2222	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2223	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2224
2225	/* Always intercept CR4 writes for tracking PGM mode changes and AVX (for
2226	XCR0 syncing during worlds switching). */
2227	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2228
2229	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2230	if (!pVM->hmr0.s.fNestedPaging)
2231	{
2232	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2233	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2234	}
2235
2236	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2237	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2238	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2239
2240	/*
2241	* Merge the guest's exception intercepts into the nested-guest VMCB.
2242	*
2243	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2244	* while executing the nested-guest.
2245	*
2246	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2247	* be tweaked later depending on how we wish to implement breakpoints.
2248	*
2249	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2250	* loaded into their guests, not ours.
2251	*
2252	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2253	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2254	*/
2255	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2256	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2257	& ~( RT_BIT(X86_XCPT_UD)
2258	\| RT_BIT(X86_XCPT_BP)
2259	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2260	#else
2261	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2262	#endif
2263
2264	/*
2265	* Adjust intercepts while executing the nested-guest that differ from the
2266	* outer guest intercepts.
2267	*
2268	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2269	* that belong to the nested-guest to the outer guest.
2270	*
2271	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2272	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2273	*/
2274	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2275	\| SVM_CTRL_INTERCEPT_VMMCALL))
2276	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2277
2278	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2279	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2280
2281	/* Finally, update the VMCB clean bits. */
2282	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2283	}
2284	#endif
2285
2286
2287	/**
2288	* Enters the AMD-V session.
2289	*
2290	* @returns VBox status code.
2291	* @param pVCpu The cross context virtual CPU structure.
2292	*/
2293	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2294	{
2295	AssertPtr(pVCpu);
2296	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2297	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2298
2299	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2300	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2301	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2302
2303	pVCpu->hmr0.s.fLeaveDone = false;
2304	return VINF_SUCCESS;
2305	}
2306
2307
2308	/**
2309	* Thread-context callback for AMD-V.
2310	*
2311	* This is used together with RTThreadCtxHookCreate() on platforms which
2312	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2313	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2314	*
2315	* @param enmEvent The thread-context event.
2316	* @param pVCpu The cross context virtual CPU structure.
2317	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2318	* @thread EMT(pVCpu)
2319	*/
2320	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2321	{
2322	NOREF(fGlobalInit);
2323
2324	switch (enmEvent)
2325	{
2326	case RTTHREADCTXEVENT_OUT:
2327	{
2328	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2329	VMCPU_ASSERT_EMT(pVCpu);
2330
2331	/* No longjmps (log-flush, locks) in this fragile context. */
2332	VMMRZCallRing3Disable(pVCpu);
2333
2334	if (!pVCpu->hmr0.s.fLeaveDone)
2335	{
2336	hmR0SvmLeave(pVCpu, false /* fImportState */);
2337	pVCpu->hmr0.s.fLeaveDone = true;
2338	}
2339
2340	/* Leave HM context, takes care of local init (term). */
2341	int rc = HMR0LeaveCpu(pVCpu);
2342	AssertRC(rc); NOREF(rc);
2343
2344	/* Restore longjmp state. */
2345	VMMRZCallRing3Enable(pVCpu);
2346	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2347	break;
2348	}
2349
2350	case RTTHREADCTXEVENT_IN:
2351	{
2352	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2353	VMCPU_ASSERT_EMT(pVCpu);
2354
2355	/* No longjmps (log-flush, locks) in this fragile context. */
2356	VMMRZCallRing3Disable(pVCpu);
2357
2358	/*
2359	* Initialize the bare minimum state required for HM. This takes care of
2360	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2361	*/
2362	int rc = hmR0EnterCpu(pVCpu);
2363	AssertRC(rc); NOREF(rc);
2364	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2365	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2366
2367	pVCpu->hmr0.s.fLeaveDone = false;
2368
2369	/* Restore longjmp state. */
2370	VMMRZCallRing3Enable(pVCpu);
2371	break;
2372	}
2373
2374	default:
2375	break;
2376	}
2377	}
2378
2379
2380	/**
2381	* Saves the host state.
2382	*
2383	* @returns VBox status code.
2384	* @param pVCpu The cross context virtual CPU structure.
2385	*
2386	* @remarks No-long-jump zone!!!
2387	*/
2388	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2389	{
2390	NOREF(pVCpu);
2391
2392	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2393	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2394	return VINF_SUCCESS;
2395	}
2396
2397
2398	/**
2399	* Exports the guest or nested-guest state from the virtual-CPU context into the
2400	* VMCB.
2401	*
2402	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2403	* code based on the virtual-CPU mode.
2404	*
2405	* @returns VBox status code.
2406	* @param pVCpu The cross context virtual CPU structure.
2407	* @param pSvmTransient Pointer to the SVM-transient structure.
2408	*
2409	* @remarks No-long-jump zone!!!
2410	*/
2411	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2412	{
2413	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2414
2415	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2416	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2417	Assert(pVmcb);
2418
2419	pVmcb->guest.u64RIP = pCtx->rip;
2420	pVmcb->guest.u64RSP = pCtx->rsp;
2421	pVmcb->guest.u64RFlags = pCtx->eflags.u;
2422	pVmcb->guest.u64RAX = pCtx->rax;
2423
2424	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2425	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2426
2427	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2428	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2429	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2430	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2431	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2432
2433	ASMSetFlags(fEFlags);
2434
2435	if (!fIsNestedGuest)
2436	{
2437	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2438	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2439	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2440	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2441	}
2442
2443	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2444	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2445	\| HM_CHANGED_GUEST_RFLAGS
2446	\| HM_CHANGED_GUEST_GPRS_MASK
2447	\| HM_CHANGED_GUEST_X87
2448	\| HM_CHANGED_GUEST_SSE_AVX
2449	\| HM_CHANGED_GUEST_OTHER_XSAVE
2450	\| HM_CHANGED_GUEST_XCRx
2451	\| HM_CHANGED_GUEST_TSC_AUX
2452	\| HM_CHANGED_GUEST_OTHER_MSRS;
2453	if (fIsNestedGuest)
2454	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2455	\| HM_CHANGED_GUEST_APIC_TPR;
2456
2457	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2458	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2459
2460	#ifdef VBOX_STRICT
2461	/*
2462	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2463	* except for the host-context and/or shared host-guest context bits.
2464	*/
2465	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2466	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2467	("fCtxChanged=%#RX64\n", fCtxChanged));
2468
2469	/*
2470	* If we need to log state that isn't always imported, we'll need to import them here.
2471	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2472	*/
2473	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2474	#endif
2475
2476	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2477	return VINF_SUCCESS;
2478	}
2479
2480	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2481
2482	/**
2483	* Merges the guest and nested-guest MSR permission bitmap.
2484	*
2485	* If the guest is intercepting an MSR we need to intercept it regardless of
2486	* whether the nested-guest is intercepting it or not.
2487	*
2488	* @param pHostCpu The HM physical-CPU structure.
2489	* @param pVCpu The cross context virtual CPU structure.
2490	*
2491	* @remarks No-long-jmp zone!!!
2492	*/
2493	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2494	{
2495	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2496	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2497	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2498
2499	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2500	uint32_t const offRsvdQwords = 0x1800 >> 3;
2501	for (uint32_t i = 0; i < offRsvdQwords; i++)
2502	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2503	}
2504
2505
2506	/**
2507	* Caches the nested-guest VMCB fields before we modify them for execution using
2508	* hardware-assisted SVM.
2509	*
2510	* @returns true if the VMCB was previously already cached, false otherwise.
2511	* @param pVCpu The cross context virtual CPU structure.
2512	*
2513	* @sa HMNotifySvmNstGstVmexit.
2514	*/
2515	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2516	{
2517	/*
2518	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2519	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2520	*
2521	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2522	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2523	*/
2524	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2525	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2526	if (!fWasCached)
2527	{
2528	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2529	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2530	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2531	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2532	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2533	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2534	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2535	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2536	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2537	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2538	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2539	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2540	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2541	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2542	pVmcbNstGstCache->fCacheValid = true;
2543	Log4Func(("Cached VMCB fields\n"));
2544	}
2545
2546	return fWasCached;
2547	}
2548
2549
2550	/**
2551	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2552	*
2553	* This is done the first time we enter nested-guest execution using SVM R0
2554	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2555	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2556	*
2557	* @param pVCpu The cross context virtual CPU structure.
2558	*/
2559	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2560	{
2561	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2562	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2563
2564	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2565
2566	/*
2567	* First cache the nested-guest VMCB fields we may potentially modify.
2568	*/
2569	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2570	if (!fVmcbCached)
2571	{
2572	/*
2573	* The IOPM of the nested-guest can be ignored because the the guest always
2574	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2575	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2576	*/
2577	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2578
2579	/*
2580	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2581	* nested-paging suddenly while executing a VM (see assertion at the end of
2582	* Trap0eHandler() in PGMAllBth.h).
2583	*/
2584	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2585
2586	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2587	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2588
2589	/*
2590	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2591	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2592	*/
2593	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2594
2595	# ifdef DEBUG_ramshankar
2596	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2597	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2598	# endif
2599
2600	/*
2601	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2602	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2603	*/
2604	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2605	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2606	\| SVM_CTRL_INTERCEPT_VMLOAD;
2607
2608	/*
2609	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2610	* CLGI/STGI instructions executed by the nested-guest.
2611	*/
2612	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2613	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2614	\| SVM_CTRL_INTERCEPT_STGI;
2615
2616	/* Merge the guest and nested-guest intercepts. */
2617	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2618
2619	/* Update the VMCB clean bits. */
2620	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2621	}
2622	else
2623	{
2624	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2625	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2626	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2627	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2628	}
2629	}
2630
2631	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2632
2633	/**
2634	* Exports the state shared between the host and guest (or nested-guest) into
2635	* the VMCB.
2636	*
2637	* @param pVCpu The cross context virtual CPU structure.
2638	* @param pVmcb Pointer to the VM control block.
2639	*
2640	* @remarks No-long-jump zone!!!
2641	*/
2642	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2643	{
2644	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2645	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2646
2647	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2648	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2649
2650	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2651	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2652	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2653	}
2654
2655
2656	/**
2657	* Worker for SVMR0ImportStateOnDemand.
2658	*
2659	* @param pVCpu The cross context virtual CPU structure.
2660	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2661	*/
2662	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2663	{
2664	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2665
2666	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2667	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2668	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2669	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2670
2671	/*
2672	* We disable interrupts to make the updating of the state and in particular
2673	* the fExtrn modification atomic wrt to preemption hooks.
2674	*/
2675	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2676
2677	fWhat &= pCtx->fExtrn;
2678	if (fWhat)
2679	{
2680	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2681	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2682	{
2683	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2684	{
2685	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2686	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2687	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2688	}
2689	}
2690
2691	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2692	{
2693	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2694	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2695	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2696	}
2697	#endif
2698
2699	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2700	CPUMUpdateInterruptShadowEx(pCtx, pVmcbCtrl->IntShadow.n.u1IntShadow, pVmcbGuest->u64RIP);
2701
2702	if (fWhat & CPUMCTX_EXTRN_RIP)
2703	pCtx->rip = pVmcbGuest->u64RIP;
2704
2705	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2706	{
2707	pCtx->eflags.u = pVmcbGuest->u64RFlags;
2708	if (pVCpu->hmr0.s.fClearTrapFlag)
2709	{
2710	pVCpu->hmr0.s.fClearTrapFlag = false;
2711	pCtx->eflags.Bits.u1TF = 0;
2712	}
2713	}
2714
2715	if (fWhat & CPUMCTX_EXTRN_RSP)
2716	pCtx->rsp = pVmcbGuest->u64RSP;
2717
2718	if (fWhat & CPUMCTX_EXTRN_RAX)
2719	pCtx->rax = pVmcbGuest->u64RAX;
2720
2721	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2722	{
2723	if (fWhat & CPUMCTX_EXTRN_CS)
2724	{
2725	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2726	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2727	/** @todo SELM might need to be fixed as it too should not care about the
2728	* granularity bit. See @bugref{6785}. */
2729	if ( !pCtx->cs.Attr.n.u1Granularity
2730	&& pCtx->cs.Attr.n.u1Present
2731	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2732	{
2733	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2734	pCtx->cs.Attr.n.u1Granularity = 1;
2735	}
2736	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2737	}
2738	if (fWhat & CPUMCTX_EXTRN_SS)
2739	{
2740	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2741	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2742	/*
2743	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2744	* VMCB and uses that and thus it's possible that when the CPL changes during
2745	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2746	* AMD Fusion CPUs with 64-bit guests.
2747	*
2748	* See AMD spec. 15.5.1 "Basic operation".
2749	*/
2750	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2751	uint8_t const uCpl = pVmcbGuest->u8CPL;
2752	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2753	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2754	}
2755	if (fWhat & CPUMCTX_EXTRN_DS)
2756	{
2757	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2758	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2759	}
2760	if (fWhat & CPUMCTX_EXTRN_ES)
2761	{
2762	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2763	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2764	}
2765	if (fWhat & CPUMCTX_EXTRN_FS)
2766	{
2767	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2768	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2769	}
2770	if (fWhat & CPUMCTX_EXTRN_GS)
2771	{
2772	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2773	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2774	}
2775	}
2776
2777	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2778	{
2779	if (fWhat & CPUMCTX_EXTRN_TR)
2780	{
2781	/*
2782	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2783	* are used between Intel and AMD, see @bugref{6208#c39}.
2784	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2785	*/
2786	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2787	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2788	{
2789	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2790	\|\| CPUMIsGuestInLongModeEx(pCtx))
2791	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2792	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2793	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2794	}
2795	}
2796
2797	if (fWhat & CPUMCTX_EXTRN_LDTR)
2798	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2799
2800	if (fWhat & CPUMCTX_EXTRN_GDTR)
2801	{
2802	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2803	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2804	}
2805
2806	if (fWhat & CPUMCTX_EXTRN_IDTR)
2807	{
2808	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2809	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2810	}
2811	}
2812
2813	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2814	{
2815	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2816	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2817	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2818	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2819	}
2820
2821	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2822	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2823	{
2824	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2825	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2826	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2827	}
2828
2829	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2830	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2831
2832	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2833	{
2834	if (fWhat & CPUMCTX_EXTRN_DR6)
2835	{
2836	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2837	pCtx->dr[6] = pVmcbGuest->u64DR6;
2838	else
2839	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2840	}
2841
2842	if (fWhat & CPUMCTX_EXTRN_DR7)
2843	{
2844	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2845	pCtx->dr[7] = pVmcbGuest->u64DR7;
2846	else
2847	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2848	}
2849	}
2850
2851	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2852	{
2853	if (fWhat & CPUMCTX_EXTRN_CR0)
2854	{
2855	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2856	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2857	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2858	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2859	CPUMSetGuestCR0(pVCpu, uCr0);
2860	VMMRZCallRing3Enable(pVCpu);
2861	}
2862
2863	if (fWhat & CPUMCTX_EXTRN_CR2)
2864	pCtx->cr2 = pVmcbGuest->u64CR2;
2865
2866	if (fWhat & CPUMCTX_EXTRN_CR3)
2867	{
2868	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2869	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2870	{
2871	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2872	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2873	}
2874	}
2875
2876	/* Changes to CR4 are always intercepted. */
2877	}
2878
2879	/* Update fExtrn. */
2880	pCtx->fExtrn &= ~fWhat;
2881
2882	/* If everything has been imported, clear the HM keeper bit. */
2883	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2884	{
2885	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2886	Assert(!pCtx->fExtrn);
2887	}
2888	}
2889	else
2890	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2891
2892	ASMSetFlags(fEFlags);
2893
2894	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2895
2896	/*
2897	* Honor any pending CR3 updates.
2898	*
2899	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2900	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2901	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2902	* handling -> hmR0SvmImportGuestState() and here we are.
2903	*
2904	* The reason for such complicated handling is because VM-exits that call into PGM expect
2905	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2906	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2907	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2908	*
2909	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2910	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2911	*/
2912	if ( VMMRZCallRing3IsEnabled(pVCpu)
2913	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2914	{
2915	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2916	PGMUpdateCR3(pVCpu, pCtx->cr3);
2917	}
2918	}
2919
2920
2921	/**
2922	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2923	* context.
2924	*
2925	* Currently there is no residual state left in the CPU that is not updated in the
2926	* VMCB.
2927	*
2928	* @returns VBox status code.
2929	* @param pVCpu The cross context virtual CPU structure.
2930	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2931	*/
2932	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2933	{
2934	hmR0SvmImportGuestState(pVCpu, fWhat);
2935	return VINF_SUCCESS;
2936	}
2937
2938
2939	/**
2940	* Gets SVM \#VMEXIT auxiliary information.
2941	*
2942	* @returns VBox status code.
2943	* @param pVCpu The cross context virtual CPU structure.
2944	* @param pSvmExitAux Where to store the auxiliary info.
2945	*/
2946	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2947	{
2948	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2949	if (RT_LIKELY(pSvmTransient))
2950	{
2951	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2952	if (RT_LIKELY(pVmcb))
2953	{
2954	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2955	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2956	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2957	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2958	return VINF_SUCCESS;
2959	}
2960	return VERR_SVM_IPE_5;
2961	}
2962	return VERR_NOT_AVAILABLE;
2963	}
2964
2965
2966	/**
2967	* Does the necessary state syncing before returning to ring-3 for any reason
2968	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2969	*
2970	* @param pVCpu The cross context virtual CPU structure.
2971	* @param fImportState Whether to import the guest state from the VMCB back
2972	* to the guest-CPU context.
2973	*
2974	* @remarks No-long-jmp zone!!!
2975	*/
2976	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2977	{
2978	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2979	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2980
2981	/*
2982	* !!! IMPORTANT !!!
2983	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2984	*/
2985
2986	/* Save the guest state if necessary. */
2987	if (fImportState)
2988	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2989
2990	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2991	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2992	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2993
2994	/*
2995	* Restore host debug registers if necessary and resync on next R0 reentry.
2996	*/
2997	#ifdef VBOX_STRICT
2998	if (CPUMIsHyperDebugStateActive(pVCpu))
2999	{
3000	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
3001	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
3002	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
3003	}
3004	#endif
3005	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3006	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3007	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3008
3009	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3010	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3011	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3012	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3013	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3014	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3015	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3016
3017	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3018	}
3019
3020
3021	/**
3022	* Leaves the AMD-V session.
3023	*
3024	* Only used while returning to ring-3 either due to longjump or exits to
3025	* ring-3.
3026	*
3027	* @returns VBox status code.
3028	* @param pVCpu The cross context virtual CPU structure.
3029	*/
3030	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3031	{
3032	HM_DISABLE_PREEMPT(pVCpu);
3033	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3034	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3035
3036	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3037	and done this from the SVMR0ThreadCtxCallback(). */
3038	if (!pVCpu->hmr0.s.fLeaveDone)
3039	{
3040	hmR0SvmLeave(pVCpu, true /* fImportState */);
3041	pVCpu->hmr0.s.fLeaveDone = true;
3042	}
3043
3044	/*
3045	* !!! IMPORTANT !!!
3046	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3047	*/
3048
3049	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3050	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3051	VMMR0ThreadCtxHookDisable(pVCpu);
3052
3053	/* Leave HM context. This takes care of local init (term). */
3054	int rc = HMR0LeaveCpu(pVCpu);
3055
3056	HM_RESTORE_PREEMPT();
3057	return rc;
3058	}
3059
3060
3061	/**
3062	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3063	* any remaining host state) before we go back to ring-3 due to an assertion.
3064	*
3065	* @param pVCpu The cross context virtual CPU structure.
3066	*/
3067	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3068	{
3069	/*
3070	* !!! IMPORTANT !!!
3071	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3072	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3073	*/
3074	VMMR0AssertionRemoveNotification(pVCpu);
3075	VMMRZCallRing3Disable(pVCpu);
3076	HM_DISABLE_PREEMPT(pVCpu);
3077
3078	/* Import the entire guest state. */
3079	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3080
3081	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3082	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3083
3084	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3085	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3086
3087	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3088	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3089	VMMR0ThreadCtxHookDisable(pVCpu);
3090
3091	/* Leave HM context. This takes care of local init (term). */
3092	HMR0LeaveCpu(pVCpu);
3093
3094	HM_RESTORE_PREEMPT();
3095	return VINF_SUCCESS;
3096	}
3097
3098
3099	/**
3100	* Take necessary actions before going back to ring-3.
3101	*
3102	* An action requires us to go back to ring-3. This function does the necessary
3103	* steps before we can safely return to ring-3. This is not the same as longjmps
3104	* to ring-3, this is voluntary.
3105	*
3106	* @returns Strict VBox status code.
3107	* @param pVCpu The cross context virtual CPU structure.
3108	* @param rcExit The reason for exiting to ring-3. Can be
3109	* VINF_VMM_UNKNOWN_RING3_CALL.
3110	*/
3111	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3112	{
3113	Assert(pVCpu);
3114	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3115
3116	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3117	VMMRZCallRing3Disable(pVCpu);
3118	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3119	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3120
3121	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3122	if (pVCpu->hm.s.Event.fPending)
3123	{
3124	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3125	Assert(!pVCpu->hm.s.Event.fPending);
3126	}
3127
3128	/* Sync. the necessary state for going back to ring-3. */
3129	hmR0SvmLeaveSession(pVCpu);
3130	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3131
3132	/* Thread-context hooks are unregistered at this point!!! */
3133	/* Ring-3 callback notifications are unregistered at this point!!! */
3134
3135	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3136	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3137	\| CPUM_CHANGED_LDTR
3138	\| CPUM_CHANGED_GDTR
3139	\| CPUM_CHANGED_IDTR
3140	\| CPUM_CHANGED_TR
3141	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3142	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3143	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3144	{
3145	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3146	}
3147
3148	/* Update the exit-to-ring 3 reason. */
3149	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3150
3151	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3152	if ( rcExit != VINF_EM_RAW_INTERRUPT
3153	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3154	{
3155	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3156	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3157	}
3158
3159	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3160	VMMRZCallRing3Enable(pVCpu);
3161
3162	/*
3163	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3164	* and if we're injecting an event we should have a TRPM trap pending.
3165	*/
3166	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3167	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3168	VERR_SVM_IPE_5);
3169	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3170	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3171	VERR_SVM_IPE_4);
3172
3173	return rcExit;
3174	}
3175
3176
3177	/**
3178	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3179	* intercepts.
3180	*
3181	* @param pVCpu The cross context virtual CPU structure.
3182	* @param pVmcb Pointer to the VM control block.
3183	*
3184	* @remarks No-long-jump zone!!!
3185	*/
3186	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3187	{
3188	/*
3189	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3190	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3191	* RDTSC/RDTSCP as well.
3192	*/
3193	bool fParavirtTsc;
3194	uint64_t uTscOffset;
3195	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3196
3197	bool fIntercept;
3198	if (fCanUseRealTsc)
3199	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3200	else
3201	{
3202	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3203	fIntercept = true;
3204	}
3205
3206	if (!fIntercept)
3207	{
3208	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3209	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3210	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3211	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3212	#endif
3213
3214	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3215	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3216	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3217	}
3218
3219	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3220	information before every VM-entry, hence we have nothing to do here at the moment. */
3221	if (fParavirtTsc)
3222	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3223	}
3224
3225
3226	/**
3227	* Sets an event as a pending event to be injected into the guest.
3228	*
3229	* @param pVCpu The cross context virtual CPU structure.
3230	* @param pEvent Pointer to the SVM event.
3231	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3232	* page-fault.
3233	*
3234	* @remarks Statistics counter assumes this is a guest event being reflected to
3235	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3236	*/
3237	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3238	{
3239	Assert(!pVCpu->hm.s.Event.fPending);
3240	Assert(pEvent->n.u1Valid);
3241
3242	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3243	pVCpu->hm.s.Event.fPending = true;
3244	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3245
3246	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3247	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3248	}
3249
3250
3251	/**
3252	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3253	*
3254	* @param pVCpu The cross context virtual CPU structure.
3255	*/
3256	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3257	{
3258	SVMEVENT Event;
3259	Event.u = 0;
3260	Event.n.u1Valid = 1;
3261	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3262	Event.n.u8Vector = X86_XCPT_DE;
3263	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3264	}
3265
3266
3267	/**
3268	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3269	*
3270	* @param pVCpu The cross context virtual CPU structure.
3271	*/
3272	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3273	{
3274	SVMEVENT Event;
3275	Event.u = 0;
3276	Event.n.u1Valid = 1;
3277	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3278	Event.n.u8Vector = X86_XCPT_UD;
3279	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3280	}
3281
3282
3283	/**
3284	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3285	*
3286	* @param pVCpu The cross context virtual CPU structure.
3287	*/
3288	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3289	{
3290	SVMEVENT Event;
3291	Event.u = 0;
3292	Event.n.u1Valid = 1;
3293	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3294	Event.n.u8Vector = X86_XCPT_DB;
3295	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3296	}
3297
3298
3299	/**
3300	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3301	*
3302	* @param pVCpu The cross context virtual CPU structure.
3303	* @param u32ErrCode The error-code for the page-fault.
3304	* @param uFaultAddress The page fault address (CR2).
3305	*
3306	* @remarks This updates the guest CR2 with @a uFaultAddress!
3307	*/
3308	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3309	{
3310	SVMEVENT Event;
3311	Event.u = 0;
3312	Event.n.u1Valid = 1;
3313	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3314	Event.n.u8Vector = X86_XCPT_PF;
3315	Event.n.u1ErrorCodeValid = 1;
3316	Event.n.u32ErrorCode = u32ErrCode;
3317
3318	/* Update CR2 of the guest. */
3319	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3320	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3321	{
3322	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3323	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3324	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3325	}
3326
3327	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3328	}
3329
3330
3331	/**
3332	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3333	*
3334	* @param pVCpu The cross context virtual CPU structure.
3335	*/
3336	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3337	{
3338	SVMEVENT Event;
3339	Event.u = 0;
3340	Event.n.u1Valid = 1;
3341	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3342	Event.n.u8Vector = X86_XCPT_MF;
3343	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3344	}
3345
3346
3347	/**
3348	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3349	*
3350	* @param pVCpu The cross context virtual CPU structure.
3351	*/
3352	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3353	{
3354	SVMEVENT Event;
3355	Event.u = 0;
3356	Event.n.u1Valid = 1;
3357	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3358	Event.n.u8Vector = X86_XCPT_DF;
3359	Event.n.u1ErrorCodeValid = 1;
3360	Event.n.u32ErrorCode = 0;
3361	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3362	}
3363
3364
3365	/**
3366	* Injects an event into the guest upon VMRUN by updating the relevant field
3367	* in the VMCB.
3368	*
3369	* @param pVCpu The cross context virtual CPU structure.
3370	* @param pVmcb Pointer to the guest VM control block.
3371	* @param pEvent Pointer to the event.
3372	*
3373	* @remarks No-long-jump zone!!!
3374	* @remarks Requires CR0!
3375	*/
3376	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3377	{
3378	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3379	pVmcb->ctrl.EventInject.u = pEvent->u;
3380	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3381	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3382	{
3383	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3384	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3385	}
3386	else
3387	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3388	RT_NOREF(pVCpu);
3389
3390	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3391	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3392	}
3393
3394
3395
3396	/**
3397	* Converts any TRPM trap into a pending HM event. This is typically used when
3398	* entering from ring-3 (not longjmp returns).
3399	*
3400	* @param pVCpu The cross context virtual CPU structure.
3401	*/
3402	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3403	{
3404	Assert(TRPMHasTrap(pVCpu));
3405	Assert(!pVCpu->hm.s.Event.fPending);
3406
3407	uint8_t uVector;
3408	TRPMEVENT enmTrpmEvent;
3409	uint32_t uErrCode;
3410	RTGCUINTPTR GCPtrFaultAddress;
3411	uint8_t cbInstr;
3412
3413	uVector = TRPMGetTrapAll(pVCpu, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3414
3415	SVMEVENT Event;
3416	Event.u = 0;
3417	Event.n.u1Valid = 1;
3418	Event.n.u8Vector = uVector;
3419
3420	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3421	if (enmTrpmEvent == TRPM_TRAP)
3422	{
3423	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3424	switch (uVector)
3425	{
3426	case X86_XCPT_BP:
3427	case X86_XCPT_OF:
3428	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3429	RT_FALL_THRU();
3430
3431	case X86_XCPT_PF:
3432	case X86_XCPT_DF:
3433	case X86_XCPT_TS:
3434	case X86_XCPT_NP:
3435	case X86_XCPT_SS:
3436	case X86_XCPT_GP:
3437	case X86_XCPT_AC:
3438	{
3439	Event.n.u1ErrorCodeValid = 1;
3440	Event.n.u32ErrorCode = uErrCode;
3441	break;
3442	}
3443	}
3444	}
3445	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3446	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3447	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3448	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3449	else if (enmTrpmEvent == TRPM_NMI)
3450	Event.n.u3Type = SVM_EVENT_NMI;
3451	else
3452	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3453
3454	int const rc = TRPMResetTrap(pVCpu);
3455	AssertRC(rc);
3456
3457	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3458	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3459
3460	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3461	}
3462
3463
3464	/**
3465	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3466	* AMD-V to execute any instruction.
3467	*
3468	* @param pVCpu The cross context virtual CPU structure.
3469	*/
3470	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3471	{
3472	Assert(pVCpu->hm.s.Event.fPending);
3473	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3474
3475	SVMEVENT Event;
3476	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3477
3478	uint8_t uVector = Event.n.u8Vector;
3479	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3480
3481	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3482
3483	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3484	AssertRC(rc);
3485
3486	if (Event.n.u1ErrorCodeValid)
3487	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3488
3489	if ( enmTrapType == TRPM_TRAP
3490	&& uVector == X86_XCPT_PF)
3491	{
3492	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3493	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3494	}
3495	else if (enmTrapType == TRPM_SOFTWARE_INT)
3496	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3497	pVCpu->hm.s.Event.fPending = false;
3498	}
3499
3500
3501	/**
3502	* Sets the virtual interrupt intercept control in the VMCB.
3503	*
3504	* @param pVCpu The cross context virtual CPU structure.
3505	* @param pVmcb Pointer to the VM control block.
3506	*/
3507	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3508	{
3509	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3510
3511	/*
3512	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3513	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3514	* (updating ISR at the right time) and inject the interrupt.
3515	*
3516	* With AVIC is supported, we could make use of the asynchronously delivery without
3517	* #VMEXIT and we would be passing the AVIC page to SVM.
3518	*
3519	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3520	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3521	*/
3522	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3523	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3524	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3525	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3526	Log4(("Set VINTR intercept\n"));
3527	}
3528
3529
3530	/**
3531	* Clears the virtual interrupt intercept control in the VMCB as
3532	* we are figured the guest is unable process any interrupts
3533	* at this point of time.
3534	*
3535	* @param pVCpu The cross context virtual CPU structure.
3536	* @param pVmcb Pointer to the VM control block.
3537	*/
3538	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3539	{
3540	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3541
3542	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3543	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3544	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3545	{
3546	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3547	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3548	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3549	Log4(("Cleared VINTR intercept\n"));
3550	}
3551	}
3552
3553
3554	/**
3555	* Evaluates the event to be delivered to the guest and sets it as the pending
3556	* event.
3557	*
3558	* @returns Strict VBox status code.
3559	* @param pVCpu The cross context virtual CPU structure.
3560	* @param pSvmTransient Pointer to the SVM transient structure.
3561	*/
3562	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3563	{
3564	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3565	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3566	\| CPUMCTX_EXTRN_RFLAGS
3567	\| CPUMCTX_EXTRN_INHIBIT_INT
3568	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3569
3570	Assert(!pVCpu->hm.s.Event.fPending);
3571	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3572	Assert(pVmcb);
3573
3574	bool const fGif = CPUMGetGuestGif(pCtx);
3575	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(pCtx);
3576	bool const fBlockNmi = CPUMAreInterruptsInhibitedByNmi(pCtx);
3577
3578	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3579	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3580	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3581
3582	/** @todo SMI. SMIs take priority over NMIs. */
3583
3584	/*
3585	* Check if the guest or nested-guest can receive NMIs.
3586	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3587	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3588	*/
3589	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3590	&& !fBlockNmi)
3591	{
3592	if ( fGif
3593	&& !fIntShadow)
3594	{
3595	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3596	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3597	{
3598	Log4(("Intercepting NMI -> #VMEXIT\n"));
3599	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3600	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3601	}
3602	#endif
3603	Log4(("Setting NMI pending for injection\n"));
3604	SVMEVENT Event;
3605	Event.u = 0;
3606	Event.n.u1Valid = 1;
3607	Event.n.u8Vector = X86_XCPT_NMI;
3608	Event.n.u3Type = SVM_EVENT_NMI;
3609	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3610	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3611	}
3612	else if (!fGif)
3613	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3614	else if (!pSvmTransient->fIsNestedGuest)
3615	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3616	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3617	}
3618	/*
3619	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3620	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3621	* it from the APIC device.
3622	*
3623	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3624	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3625	* do that work when we execute nested-guest code esp. since all the required information
3626	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3627	* the virtual interrupt controller.
3628	*
3629	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3630	*/
3631	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3632	&& !pVCpu->hm.s.fSingleInstruction)
3633	{
3634	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u & X86_EFL_IF)
3635	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3636	if ( fGif
3637	&& !fBlockInt
3638	&& !fIntShadow)
3639	{
3640	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3641	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3642	{
3643	Log4(("Intercepting INTR -> #VMEXIT\n"));
3644	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3645	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3646	}
3647	#endif
3648	uint8_t u8Interrupt;
3649	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3650	if (RT_SUCCESS(rc))
3651	{
3652	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3653	SVMEVENT Event;
3654	Event.u = 0;
3655	Event.n.u1Valid = 1;
3656	Event.n.u8Vector = u8Interrupt;
3657	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3658	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3659	}
3660	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3661	{
3662	/*
3663	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3664	* updated eventually when the TPR is written by the guest.
3665	*/
3666	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3667	}
3668	else
3669	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3670	}
3671	else if (!fGif)
3672	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3673	else if (!pSvmTransient->fIsNestedGuest)
3674	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3675	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3676	}
3677
3678	return VINF_SUCCESS;
3679	}
3680
3681
3682	/**
3683	* Injects any pending events into the guest (or nested-guest).
3684	*
3685	* @param pVCpu The cross context virtual CPU structure.
3686	* @param pVmcb Pointer to the VM control block.
3687	*
3688	* @remarks Must only be called when we are guaranteed to enter
3689	* hardware-assisted SVM execution and not return to ring-3
3690	* prematurely.
3691	*/
3692	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3693	{
3694	Assert(!TRPMHasTrap(pVCpu));
3695	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3696
3697	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx);
3698	#ifdef VBOX_STRICT
3699	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3700	bool const fGif = CPUMGetGuestGif(pCtx);
3701	bool fAllowInt = fGif;
3702	if (fGif)
3703	{
3704	/*
3705	* For nested-guests we have no way to determine if we're injecting a physical or
3706	* virtual interrupt at this point. Hence the partial verification below.
3707	*/
3708	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3709	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3710	else
3711	fAllowInt = RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
3712	}
3713	#endif
3714
3715	if (pVCpu->hm.s.Event.fPending)
3716	{
3717	SVMEVENT Event;
3718	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3719	Assert(Event.n.u1Valid);
3720
3721	/*
3722	* Validate event injection pre-conditions.
3723	*/
3724	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3725	{
3726	Assert(fAllowInt);
3727	Assert(!fIntShadow);
3728	}
3729	else if (Event.n.u3Type == SVM_EVENT_NMI)
3730	{
3731	Assert(fGif);
3732	Assert(!fIntShadow);
3733	}
3734
3735	/*
3736	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3737	* do this only when we are surely going to inject the NMI as otherwise if we return
3738	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3739	* SVM R0.
3740	*
3741	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3742	* which will set the VMCS field after actually delivering the NMI which we read on
3743	* VM-exit to determine the state.
3744	*/
3745	if ( Event.n.u3Type == SVM_EVENT_NMI
3746	&& Event.n.u8Vector == X86_XCPT_NMI)
3747	CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3748
3749	/*
3750	* Inject it (update VMCB for injection by the hardware).
3751	*/
3752	Log4(("Injecting pending HM event\n"));
3753	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3754	pVCpu->hm.s.Event.fPending = false;
3755
3756	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3757	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3758	else
3759	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3760	}
3761	else
3762	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3763
3764	/*
3765	* We could have injected an NMI through IEM and continue guest execution using
3766	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3767	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3768	*/
3769	if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
3770	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3771
3772	/*
3773	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3774	*
3775	* For nested-guests: We need to update it too for the scenario where IEM executes
3776	* the nested-guest but execution later continues here with an interrupt shadow active.
3777	*/
3778	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3779	}
3780
3781
3782	/**
3783	* Reports world-switch error and dumps some useful debug info.
3784	*
3785	* @param pVCpu The cross context virtual CPU structure.
3786	* @param rcVMRun The return code from VMRUN (or
3787	* VERR_SVM_INVALID_GUEST_STATE for invalid
3788	* guest-state).
3789	*/
3790	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3791	{
3792	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3793	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3794	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3795
3796	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3797	{
3798	#ifdef VBOX_STRICT
3799	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3800	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3801	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3802	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3803	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3804	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3805	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3806	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3807	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3808	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3809	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3810	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3811
3812	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3813	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3814	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3815
3816	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3817	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3818	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3819	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3820	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3821	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3822	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3823	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3824	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3825	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3826	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3827	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3828
3829	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3830	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3831	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3832	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3833	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3834	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3835	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3836	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3837	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3838	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3839	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3840	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3841	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3842	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3843	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3844	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3845	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3846	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3847	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3848	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3849
3850	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3851
3852	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3853	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3854
3855	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3856	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3857	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3858	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3859	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3860	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3861	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3862	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3863	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3864	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3865	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3866	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3867	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3868	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3869	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3870	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3871	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3872	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3873	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3874	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3875
3876	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3877	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3878
3879	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3880	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3881	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3882	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3883
3884	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3885	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3886
3887	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3888	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3889	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3890	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3891
3892	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3893	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3894	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3895	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3896	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3897	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3898	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3899
3900	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3901	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3902	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3903	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3904
3905	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3906	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3907	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3908
3909	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3910	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3911	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3912	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3913	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3914	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3915	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3916	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3917	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3918	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3919	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3920	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3921
3922	NOREF(pVmcb);
3923	#endif /* VBOX_STRICT */
3924	}
3925	else
3926	Log4Func(("rcVMRun=%d\n", rcVMRun));
3927	}
3928
3929
3930	/**
3931	* Check per-VM and per-VCPU force flag actions that require us to go back to
3932	* ring-3 for one reason or another.
3933	*
3934	* @returns Strict VBox status code (information status code included).
3935	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3936	* ring-3.
3937	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3938	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3939	* interrupts)
3940	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3941	* all EMTs to be in ring-3.
3942	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3943	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3944	* to the EM loop.
3945	*
3946	* @param pVCpu The cross context virtual CPU structure.
3947	*/
3948	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3949	{
3950	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3951
3952	/* Could happen as a result of longjump. */
3953	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3954	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3955
3956	/* Update pending interrupts into the APIC's IRR. */
3957	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3958	PDMApicUpdatePendingInterrupts(pVCpu);
3959
3960	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3961	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3962	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3963	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3964	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3965	{
3966	/* Pending PGM C3 sync. */
3967	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3968	{
3969	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3970	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3971	if (rc != VINF_SUCCESS)
3972	{
3973	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3974	return rc;
3975	}
3976	}
3977
3978	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3979	/* -XXX- what was that about single stepping? */
3980	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3981	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3982	{
3983	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3984	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3985	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3986	return rc;
3987	}
3988
3989	/* Pending VM request packets, such as hardware interrupts. */
3990	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3991	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3992	{
3993	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
3994	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3995	return VINF_EM_PENDING_REQUEST;
3996	}
3997
3998	/* Pending PGM pool flushes. */
3999	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4000	{
4001	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4002	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4003	return VINF_PGM_POOL_FLUSH_PENDING;
4004	}
4005
4006	/* Pending DMA requests. */
4007	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4008	{
4009	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4010	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4011	return VINF_EM_RAW_TO_R3;
4012	}
4013	}
4014
4015	return VINF_SUCCESS;
4016	}
4017
4018
4019	/**
4020	* Does the preparations before executing guest code in AMD-V.
4021	*
4022	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4023	* recompiler. We must be cautious what we do here regarding committing
4024	* guest-state information into the VMCB assuming we assuredly execute the guest
4025	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4026	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4027	* that the recompiler can (and should) use them when it resumes guest
4028	* execution. Otherwise such operations must be done when we can no longer
4029	* exit to ring-3.
4030	*
4031	* @returns Strict VBox status code (informational status codes included).
4032	* @retval VINF_SUCCESS if we can proceed with running the guest.
4033	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4034	*
4035	* @param pVCpu The cross context virtual CPU structure.
4036	* @param pSvmTransient Pointer to the SVM transient structure.
4037	*/
4038	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4039	{
4040	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4041
4042	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4043	if (pSvmTransient->fIsNestedGuest)
4044	{
4045	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4046	return VINF_EM_RESCHEDULE_REM;
4047	}
4048	#endif
4049
4050	/* Check force flag actions that might require us to go back to ring-3. */
4051	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4052	if (rc != VINF_SUCCESS)
4053	return rc;
4054
4055	if (TRPMHasTrap(pVCpu))
4056	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4057	else if (!pVCpu->hm.s.Event.fPending)
4058	{
4059	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4060	if ( rc != VINF_SUCCESS
4061	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4062	{
4063	/* If a nested-guest VM-exit occurred, bail. */
4064	if (pSvmTransient->fIsNestedGuest)
4065	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4066	return rc;
4067	}
4068	}
4069
4070	/*
4071	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4072	* Just do it in software, see @bugref{8411}.
4073	* NB: If we could continue a task switch exit we wouldn't need to do this.
4074	*/
4075	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4076	if (RT_UNLIKELY( !g_fHmSvmFeatures
4077	&& pVCpu->hm.s.Event.fPending
4078	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4079	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4080
4081	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4082	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4083	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4084	#endif
4085
4086	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4087	/*
4088	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4089	*/
4090	if (pSvmTransient->fIsNestedGuest)
4091	hmR0SvmSetupVmcbNested(pVCpu);
4092	#endif
4093
4094	/*
4095	* Export the guest state bits that are not shared with the host in any way as we can
4096	* longjmp or get preempted in the midst of exporting some of the state.
4097	*/
4098	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4099	AssertRCReturn(rc, rc);
4100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4101
4102	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4103	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4104
4105	/*
4106	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4107	* world-switch so we can update it on the way back if the guest changed the TPR.
4108	*/
4109	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4110	{
4111	Assert(!pSvmTransient->fIsNestedGuest);
4112	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4113	if (pVM->hm.s.fTprPatchingActive)
4114	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4115	else
4116	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4117	}
4118
4119	/*
4120	* No longjmps to ring-3 from this point on!!!
4121	*
4122	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4123	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4124	*/
4125	VMMRZCallRing3Disable(pVCpu);
4126
4127	/*
4128	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4129	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4130	* preemption disabled for a while. Since this is purly to aid the
4131	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4132	* disable interrupt on NT.
4133	*
4134	* We need to check for force-flags that could've possible been altered since we last
4135	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4136	* see @bugref{6398}).
4137	*
4138	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4139	* to ring-3 before executing guest code.
4140	*/
4141	pSvmTransient->fEFlags = ASMIntDisableFlags();
4142	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4143	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4144	{
4145	ASMSetFlags(pSvmTransient->fEFlags);
4146	VMMRZCallRing3Enable(pVCpu);
4147	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4148	return VINF_EM_RAW_TO_R3;
4149	}
4150	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4151	{
4152	ASMSetFlags(pSvmTransient->fEFlags);
4153	VMMRZCallRing3Enable(pVCpu);
4154	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4155	return VINF_EM_RAW_INTERRUPT;
4156	}
4157
4158	return VINF_SUCCESS;
4159	}
4160
4161
4162	/**
4163	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4164	* doing so.
4165	*
4166	* This means there is no backing out to ring-3 or anywhere else at this point.
4167	*
4168	* @param pVCpu The cross context virtual CPU structure.
4169	* @param pSvmTransient Pointer to the SVM transient structure.
4170	*
4171	* @remarks Called with preemption disabled.
4172	* @remarks No-long-jump zone!!!
4173	*/
4174	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4175	{
4176	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4177	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4178
4179	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4180	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4181
4182	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4183	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4184
4185	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4186
4187	if (!CPUMIsGuestFPUStateActive(pVCpu))
4188	{
4189	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4190	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4191	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4192	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4193	}
4194
4195	/* Load the state shared between host and guest (FPU, debug). */
4196	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4197	hmR0SvmExportSharedState(pVCpu, pVmcb);
4198
4199	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4200	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4201
4202	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4203	RTCPUID const idHostCpu = pHostCpu->idCpu;
4204	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4205
4206	/* Setup TSC offsetting. */
4207	if ( pSvmTransient->fUpdateTscOffsetting
4208	\|\| fMigratedHostCpu)
4209	{
4210	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4211	pSvmTransient->fUpdateTscOffsetting = false;
4212	}
4213
4214	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4215	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4216	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4217	else
4218	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4219
4220	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4221	if (fMigratedHostCpu)
4222	pVmcb->ctrl.u32VmcbCleanBits = 0;
4223
4224	/* Store status of the shared guest-host state at the time of VMRUN. */
4225	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4226	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4227
4228	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4229	uint8_t *pbMsrBitmap;
4230	if (!pSvmTransient->fIsNestedGuest)
4231	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4232	else
4233	{
4234	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4235	* MSRPM and only perform this if it changed also use EVEX.POR when it
4236	* does. */
4237	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4238
4239	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4240	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4241	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4242	}
4243	#else
4244	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4245	#endif
4246
4247	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4248	/* Flush the appropriate tagged-TLB entries. */
4249	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4250	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4251
4252	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4253
4254	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4255	to start executing. */
4256
4257	/*
4258	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4259	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4260	*
4261	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4262	*/
4263	if ( g_CpumHostFeatures.s.fRdTscP
4264	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4265	{
4266	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4267	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4268	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4269	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4270	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4271	pSvmTransient->fRestoreTscAuxMsr = true;
4272	}
4273	else
4274	{
4275	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4276	pSvmTransient->fRestoreTscAuxMsr = false;
4277	}
4278	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4279
4280	/*
4281	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4282	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4283	* from the VMCB.
4284	*/
4285	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4286	if (!fSupportsVmcbCleanBits)
4287	pVmcb->ctrl.u32VmcbCleanBits = 0;
4288	}
4289
4290
4291	/**
4292	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4293	*
4294	* @returns VBox strict status code.
4295	* @param pVCpu The cross context virtual CPU structure.
4296	* @param HCPhysVmcb The host physical address of the VMCB.
4297	*
4298	* @remarks No-long-jump zone!!!
4299	*/
4300	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4301	{
4302	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4303	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4304	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4305	}
4306
4307
4308	/**
4309	* Performs some essential restoration of state after running guest (or
4310	* nested-guest) code in AMD-V.
4311	*
4312	* @param pVCpu The cross context virtual CPU structure.
4313	* @param pSvmTransient Pointer to the SVM transient structure.
4314	* @param rcVMRun Return code of VMRUN.
4315	*
4316	* @remarks Called with interrupts disabled.
4317	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4318	* unconditionally when it is safe to do so.
4319	*/
4320	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4321	{
4322	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4323
4324	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4325	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4326
4327	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4328	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4329
4330	/* TSC read must be done early for maximum accuracy. */
4331	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4332	{
4333	if (!pSvmTransient->fIsNestedGuest)
4334	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4335	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4336	else
4337	{
4338	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4339	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4340	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4341	}
4342	#endif
4343	}
4344
4345	if (pSvmTransient->fRestoreTscAuxMsr)
4346	{
4347	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4348	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4349	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4350	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4351	}
4352
4353	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4354	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4355	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4356	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4357
4358	Assert(!(ASMGetFlags() & X86_EFL_IF));
4359	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4360	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4361
4362	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4363	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4364	{
4365	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4366	return;
4367	}
4368
4369	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4370	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4371	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4372	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4373
4374	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4375	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4376	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4377	#else
4378	/*
4379	* Always import the following:
4380	*
4381	* - RIP for exit optimizations and evaluating event injection on re-entry.
4382	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4383	* state on preemption.
4384	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4385	* - CS for exit optimizations.
4386	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4387	* assembly switcher code.
4388	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4389	*/
4390	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4391	\| CPUMCTX_EXTRN_RFLAGS
4392	\| CPUMCTX_EXTRN_RAX
4393	\| CPUMCTX_EXTRN_RSP
4394	\| CPUMCTX_EXTRN_CS
4395	\| CPUMCTX_EXTRN_HWVIRT
4396	\| CPUMCTX_EXTRN_INHIBIT_INT
4397	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4398	\| HMSVM_CPUMCTX_SHARED_STATE);
4399	#endif
4400
4401	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4402	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4403	{
4404	Assert(!pSvmTransient->fIsNestedGuest);
4405	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4406	if ( pVM->hm.s.fTprPatchingActive
4407	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4408	{
4409	int rc = PDMApicSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4410	AssertRC(rc);
4411	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4412	}
4413	/* Sync TPR when we aren't intercepting CR8 writes. */
4414	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4415	{
4416	int rc = PDMApicSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4417	AssertRC(rc);
4418	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4419	}
4420	}
4421
4422	#ifdef DEBUG_ramshankar
4423	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4424	{
4425	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4426	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4427	0 /* uVerbose */);
4428	}
4429	#endif
4430
4431	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4432	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4433	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4434	}
4435
4436
4437	/**
4438	* Runs the guest code using AMD-V.
4439	*
4440	* @returns Strict VBox status code.
4441	* @param pVCpu The cross context virtual CPU structure.
4442	* @param pcLoops Pointer to the number of executed loops.
4443	*/
4444	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4445	{
4446	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4447	Assert(pcLoops);
4448	Assert(*pcLoops <= cMaxResumeLoops);
4449
4450	SVMTRANSIENT SvmTransient;
4451	RT_ZERO(SvmTransient);
4452	SvmTransient.fUpdateTscOffsetting = true;
4453	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4454
4455	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4456	for (;;)
4457	{
4458	Assert(!HMR0SuspendPending());
4459	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4460
4461	/* Preparatory work for running nested-guest code, this may force us to return to
4462	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4463	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4464	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4465	if (rc != VINF_SUCCESS)
4466	break;
4467
4468	/*
4469	* No longjmps to ring-3 from this point on!!!
4470	*
4471	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4472	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4473	*/
4474	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4475	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4476
4477	/* Restore any residual host-state and save any bits shared between host and guest
4478	into the guest-CPU state. Re-enables interrupts! */
4479	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4480
4481	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4482	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4483	{
4484	if (rc == VINF_SUCCESS)
4485	rc = VERR_SVM_INVALID_GUEST_STATE;
4486	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4487	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4488	break;
4489	}
4490
4491	/* Handle the #VMEXIT. */
4492	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4493	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4494	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4495	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4496	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4497	if (rc != VINF_SUCCESS)
4498	break;
4499	if (++(*pcLoops) >= cMaxResumeLoops)
4500	{
4501	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4502	rc = VINF_EM_RAW_INTERRUPT;
4503	break;
4504	}
4505	}
4506
4507	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4508	return rc;
4509	}
4510
4511
4512	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4513	/**
4514	* Runs the nested-guest code using AMD-V.
4515	*
4516	* @returns Strict VBox status code.
4517	* @param pVCpu The cross context virtual CPU structure.
4518	* @param pcLoops Pointer to the number of executed loops. If we're switching
4519	* from the guest-code execution loop to this nested-guest
4520	* execution loop pass the remainder value, else pass 0.
4521	*/
4522	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4523	{
4524	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4525	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4526	Assert(pcLoops);
4527	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4528	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4529	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4530
4531	SVMTRANSIENT SvmTransient;
4532	RT_ZERO(SvmTransient);
4533	SvmTransient.fUpdateTscOffsetting = true;
4534	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4535	SvmTransient.fIsNestedGuest = true;
4536
4537	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4538	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4539
4540	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4541	for (;;)
4542	{
4543	Assert(!HMR0SuspendPending());
4544	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4545
4546	/* Preparatory work for running nested-guest code, this may force us to return to
4547	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4548	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4549	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4550	if ( rc != VINF_SUCCESS
4551	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4552	break;
4553
4554	/*
4555	* No longjmps to ring-3 from this point on!!!
4556	*
4557	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4558	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4559	*/
4560	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4561
4562	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4563
4564	/* Restore any residual host-state and save any bits shared between host and guest
4565	into the guest-CPU state. Re-enables interrupts! */
4566	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4567
4568	if (RT_LIKELY( rc == VINF_SUCCESS
4569	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4570	{ /* extremely likely */ }
4571	else
4572	{
4573	/* VMRUN failed, shouldn't really happen, Guru. */
4574	if (rc != VINF_SUCCESS)
4575	break;
4576
4577	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4578	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4579	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4580	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4581	break;
4582	}
4583
4584	/* Handle the #VMEXIT. */
4585	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4586	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4587	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4588	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4589	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4590	if (rc == VINF_SUCCESS)
4591	{
4592	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4593	{
4594	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4595	rc = VINF_SVM_VMEXIT;
4596	}
4597	else
4598	{
4599	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4600	continue;
4601	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4602	rc = VINF_EM_RAW_INTERRUPT;
4603	}
4604	}
4605	else
4606	Assert(rc != VINF_SVM_VMEXIT);
4607	break;
4608	/** @todo NSTSVM: handle single-stepping. */
4609	}
4610
4611	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4612	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4613
4614	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4615	return rc;
4616	}
4617	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4618
4619
4620	/**
4621	* Checks if any expensive dtrace probes are enabled and we should go to the
4622	* debug loop.
4623	*
4624	* @returns true if we should use debug loop, false if not.
4625	*/
4626	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4627	{
4628	/* It's probably faster to OR the raw 32-bit counter variables together.
4629	Since the variables are in an array and the probes are next to one
4630	another (more or less), we have good locality. So, better read
4631	eight-nine cache lines ever time and only have one conditional, than
4632	128+ conditionals, right? */
4633	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4634	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4635	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4636	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4637	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4638	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4639	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4640	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4641	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4642	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4643	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4644	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4645	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4646	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4647	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4648	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4649	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4650	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4651	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4652	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4653	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4654	) != 0
4655	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4656	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4657	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4658	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4659	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4660	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4661	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4662	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4663	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4664	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4665	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4666	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4673	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4675	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4676	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4677	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4678	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4679	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4680	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4681	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4682	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4683	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4684	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4685	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4686	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4687	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4689	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4690	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4691	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4692	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4693	) != 0
4694	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4695	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4696	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4697	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4698	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4699	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4700	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4701	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4702	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4703	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4704	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4705	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4713	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4715	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4716	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4717	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4718	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4719	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4720	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4721	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4722	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4723	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4724	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4725	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4726	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4727	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4729	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4730	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4731	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4732	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4733	) != 0;
4734	}
4735
4736
4737	/**
4738	* Runs the guest code using AMD-V.
4739	*
4740	* @returns Strict VBox status code.
4741	* @param pVCpu The cross context virtual CPU structure.
4742	*/
4743	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4744	{
4745	AssertPtr(pVCpu);
4746	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4747	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4748	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4749	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4750
4751	uint32_t cLoops = 0;
4752	VBOXSTRICTRC rc;
4753	for (;;)
4754	{
4755	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4756	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4757	#else
4758	NOREF(pCtx);
4759	bool const fInNestedGuestMode = false;
4760	#endif
4761	if (!fInNestedGuestMode)
4762	{
4763	if ( !pVCpu->hm.s.fUseDebugLoop
4764	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4765	&& !DBGFIsStepping(pVCpu)
4766	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledSwBreakpoints)
4767	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4768	else
4769	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4770	}
4771	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4772	else
4773	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4774
4775	if (rc == VINF_SVM_VMRUN)
4776	{
4777	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4778	continue;
4779	}
4780	if (rc == VINF_SVM_VMEXIT)
4781	{
4782	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4783	continue;
4784	}
4785	#endif
4786	break;
4787	}
4788
4789	/* Fixup error codes. */
4790	if (rc == VERR_EM_INTERPRETER)
4791	rc = VINF_EM_RAW_EMULATE_INSTR;
4792	else if (rc == VINF_EM_RESET)
4793	rc = VINF_EM_TRIPLE_FAULT;
4794
4795	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4796	rc = hmR0SvmExitToRing3(pVCpu, rc);
4797	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4798	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4799	return rc;
4800	}
4801
4802	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4803
4804	/**
4805	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4806	*
4807	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4808	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4809	*/
4810	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4811	{
4812	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4813	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4814	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4815	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4816	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4817	const bool fRep = pIoExitInfo->n.u1Rep;
4818	const bool fStrIo = pIoExitInfo->n.u1Str;
4819
4820	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4821	NULL /* pIoExitInfo */);
4822	}
4823
4824
4825	/**
4826	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4827	* SVM_EXIT_INVALID).
4828	*
4829	* @returns VBox status code (informational status codes included).
4830	* @param pVCpu The cross context virtual CPU structure.
4831	* @param pSvmTransient Pointer to the SVM transient structure.
4832	*/
4833	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4834	{
4835	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4836	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4837	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4838
4839	/*
4840	* We import the complete state here because we use separate VMCBs for the guest and the
4841	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4842	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4843	*/
4844	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4845	do { \
4846	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4847	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4848	} while (0)
4849
4850	/*
4851	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4852	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4853	*/
4854	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4855	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4856	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4857	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4858	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4859	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4860
4861	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4862	switch (uExitCode)
4863	{
4864	case SVM_EXIT_CPUID:
4865	{
4866	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4867	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4868	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4869	}
4870
4871	case SVM_EXIT_RDTSC:
4872	{
4873	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4874	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4875	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4876	}
4877
4878	case SVM_EXIT_RDTSCP:
4879	{
4880	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4881	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4882	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4883	}
4884
4885	case SVM_EXIT_MONITOR:
4886	{
4887	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4888	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4889	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4890	}
4891
4892	case SVM_EXIT_MWAIT:
4893	{
4894	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4895	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4896	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4897	}
4898
4899	case SVM_EXIT_HLT:
4900	{
4901	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4902	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4903	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4904	}
4905
4906	case SVM_EXIT_MSR:
4907	{
4908	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4909	{
4910	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4911	uint16_t offMsrpm;
4912	uint8_t uMsrpmBit;
4913	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4914	if (RT_SUCCESS(rc))
4915	{
4916	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4917	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4918
4919	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4920	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4921	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4922
4923	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4924	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4925	{
4926	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4927	}
4928	}
4929	else
4930	{
4931	/*
4932	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4933	* See AMD-V spec. "15.11 MSR Intercepts".
4934	*/
4935	Assert(rc == VERR_OUT_OF_RANGE);
4936	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4937	}
4938	}
4939	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4940	}
4941
4942	case SVM_EXIT_IOIO:
4943	{
4944	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4945	{
4946	SVMIOIOEXITINFO IoExitInfo;
4947	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4948	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4949	if (fIntercept)
4950	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4951	}
4952	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4953	}
4954
4955	case SVM_EXIT_XCPT_PF:
4956	{
4957	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4958	if (pVM->hmr0.s.fNestedPaging)
4959	{
4960	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4961	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4962
4963	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4964	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4965	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4966
4967	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4968	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4969	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4970	return VINF_SUCCESS;
4971	}
4972	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4973	}
4974
4975	case SVM_EXIT_XCPT_UD:
4976	{
4977	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4978	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4979	hmR0SvmSetPendingXcptUD(pVCpu);
4980	return VINF_SUCCESS;
4981	}
4982
4983	case SVM_EXIT_XCPT_MF:
4984	{
4985	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4986	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4987	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4988	}
4989
4990	case SVM_EXIT_XCPT_DB:
4991	{
4992	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4993	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4994	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
4995	}
4996
4997	case SVM_EXIT_XCPT_AC:
4998	{
4999	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5000	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5001	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5002	}
5003
5004	case SVM_EXIT_XCPT_BP:
5005	{
5006	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5007	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5008	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5009	}
5010
5011	case SVM_EXIT_READ_CR0:
5012	case SVM_EXIT_READ_CR3:
5013	case SVM_EXIT_READ_CR4:
5014	{
5015	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5016	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5017	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5018	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5019	}
5020
5021	case SVM_EXIT_CR0_SEL_WRITE:
5022	{
5023	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5024	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5025	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5026	}
5027
5028	case SVM_EXIT_WRITE_CR0:
5029	case SVM_EXIT_WRITE_CR3:
5030	case SVM_EXIT_WRITE_CR4:
5031	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5032	{
5033	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5034	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5035
5036	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5037	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5038	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5039	}
5040
5041	case SVM_EXIT_PAUSE:
5042	{
5043	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5044	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5045	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5046	}
5047
5048	case SVM_EXIT_VINTR:
5049	{
5050	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5051	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5052	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5053	}
5054
5055	case SVM_EXIT_INTR:
5056	case SVM_EXIT_NMI:
5057	case SVM_EXIT_SMI:
5058	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5059	{
5060	/*
5061	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5062	*
5063	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5064	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5065	* nested-guest #VMEXIT.
5066	*
5067	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5068	* while trying to inject interrupts, see comment at the top of this function.
5069	*/
5070	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5071	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5072	}
5073
5074	case SVM_EXIT_FERR_FREEZE:
5075	{
5076	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5077	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5078	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5079	}
5080
5081	case SVM_EXIT_INVLPG:
5082	{
5083	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5084	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5085	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5086	}
5087
5088	case SVM_EXIT_WBINVD:
5089	{
5090	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5091	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5092	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5093	}
5094
5095	case SVM_EXIT_INVD:
5096	{
5097	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5098	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5099	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5100	}
5101
5102	case SVM_EXIT_RDPMC:
5103	{
5104	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5105	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5106	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5107	}
5108
5109	default:
5110	{
5111	switch (uExitCode)
5112	{
5113	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5114	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5115	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5116	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5117	{
5118	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5119	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5120	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5121	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5122	}
5123
5124	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5125	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5126	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5127	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5128	{
5129	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5130	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5131	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5132	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5133	}
5134
5135	case SVM_EXIT_XCPT_DE:
5136	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5137	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5138	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5139	case SVM_EXIT_XCPT_OF:
5140	case SVM_EXIT_XCPT_BR:
5141	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5142	case SVM_EXIT_XCPT_NM:
5143	case SVM_EXIT_XCPT_DF:
5144	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5145	case SVM_EXIT_XCPT_TS:
5146	case SVM_EXIT_XCPT_NP:
5147	case SVM_EXIT_XCPT_SS:
5148	case SVM_EXIT_XCPT_GP:
5149	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5150	case SVM_EXIT_XCPT_15: /* Reserved. */
5151	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5152	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5153	case SVM_EXIT_XCPT_MC:
5154	case SVM_EXIT_XCPT_XF:
5155	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5156	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5157	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5158	{
5159	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5160	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5161	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5162	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5163	}
5164
5165	case SVM_EXIT_XSETBV:
5166	{
5167	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5168	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5169	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5170	}
5171
5172	case SVM_EXIT_TASK_SWITCH:
5173	{
5174	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5175	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5176	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5177	}
5178
5179	case SVM_EXIT_IRET:
5180	{
5181	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5182	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5183	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5184	}
5185
5186	case SVM_EXIT_SHUTDOWN:
5187	{
5188	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5189	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5190	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5191	}
5192
5193	case SVM_EXIT_VMMCALL:
5194	{
5195	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5196	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5197	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5198	}
5199
5200	case SVM_EXIT_CLGI:
5201	{
5202	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5203	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5204	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5205	}
5206
5207	case SVM_EXIT_STGI:
5208	{
5209	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5210	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5211	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5212	}
5213
5214	case SVM_EXIT_VMLOAD:
5215	{
5216	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5217	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5218	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5219	}
5220
5221	case SVM_EXIT_VMSAVE:
5222	{
5223	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5224	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5225	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5226	}
5227
5228	case SVM_EXIT_INVLPGA:
5229	{
5230	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5231	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5232	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5233	}
5234
5235	case SVM_EXIT_VMRUN:
5236	{
5237	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5238	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5239	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5240	}
5241
5242	case SVM_EXIT_RSM:
5243	{
5244	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5245	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5246	hmR0SvmSetPendingXcptUD(pVCpu);
5247	return VINF_SUCCESS;
5248	}
5249
5250	case SVM_EXIT_SKINIT:
5251	{
5252	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5253	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5254	hmR0SvmSetPendingXcptUD(pVCpu);
5255	return VINF_SUCCESS;
5256	}
5257
5258	case SVM_EXIT_NPF:
5259	{
5260	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5261	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5262	}
5263
5264	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5265	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5266
5267	default:
5268	{
5269	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5270	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5271	return VERR_SVM_UNKNOWN_EXIT;
5272	}
5273	}
5274	}
5275	}
5276	/* not reached */
5277
5278	# undef NST_GST_VMEXIT_CALL_RET
5279	}
5280
5281	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5282
5283	/** @def VMEXIT_CALL_RET
5284	* Used by hmR0SvmHandleExit and hmR0SvmDebugHandleExit
5285	*/
5286	#ifdef DEBUG_ramshankar
5287	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5288	do { \
5289	if ((a_fDbg) == 1) \
5290	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5291	int rc = a_CallExpr; \
5292	if ((a_fDbg) == 1) \
5293	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5294	return rc; \
5295	} while (0)
5296	#else
5297	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5298	#endif
5299
5300	/**
5301	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5302	*
5303	* @returns Strict VBox status code (informational status codes included).
5304	* @param pVCpu The cross context virtual CPU structure.
5305	* @param pSvmTransient Pointer to the SVM transient structure.
5306	*/
5307	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5308	{
5309	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5310	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5311
5312	/*
5313	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5314	* for most guests under normal workloads (for some definition of "normal").
5315	*/
5316	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5317	switch (uExitCode)
5318	{
5319	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5320	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5321	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5322	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5323	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5324	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5325	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5326	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5327	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5328	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5329
5330	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5331	case SVM_EXIT_INTR:
5332	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5333
5334	case SVM_EXIT_READ_CR0:
5335	case SVM_EXIT_READ_CR3:
5336	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5337
5338	case SVM_EXIT_CR0_SEL_WRITE:
5339	case SVM_EXIT_WRITE_CR0:
5340	case SVM_EXIT_WRITE_CR3:
5341	case SVM_EXIT_WRITE_CR4:
5342	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5343
5344	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5345	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5346	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5347	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5348	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5349	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5350	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5351	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5352	case SVM_EXIT_XCPT_DE: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDE(pVCpu, pSvmTransient));
5353	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5354	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5355	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5356	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5357	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5358	case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5359	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5360	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5361
5362	default:
5363	{
5364	switch (pSvmTransient->u64ExitCode)
5365	{
5366	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5367	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5368	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5369	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5370	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5371
5372	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5373	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5374	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5375	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5376	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5377
5378	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5379	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5380
5381	case SVM_EXIT_SMI:
5382	case SVM_EXIT_INIT:
5383	{
5384	/*
5385	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5386	* If it ever does, we want to know about it so log the exit code and bail.
5387	*/
5388	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5389	}
5390
5391	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5392	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5393	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5394	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5395	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5396	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5397	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5398	#else
5399	case SVM_EXIT_CLGI:
5400	case SVM_EXIT_STGI:
5401	case SVM_EXIT_VMLOAD:
5402	case SVM_EXIT_VMSAVE:
5403	case SVM_EXIT_INVLPGA:
5404	case SVM_EXIT_VMRUN:
5405	#endif
5406	case SVM_EXIT_RSM:
5407	case SVM_EXIT_SKINIT:
5408	{
5409	hmR0SvmSetPendingXcptUD(pVCpu);
5410	return VINF_SUCCESS;
5411	}
5412
5413	/*
5414	* The remaining should only be possible when debugging or dtracing.
5415	*/
5416	case SVM_EXIT_XCPT_DE:
5417	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5418	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5419	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5420	case SVM_EXIT_XCPT_OF:
5421	case SVM_EXIT_XCPT_BR:
5422	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5423	case SVM_EXIT_XCPT_NM:
5424	case SVM_EXIT_XCPT_DF:
5425	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5426	case SVM_EXIT_XCPT_TS:
5427	case SVM_EXIT_XCPT_NP:
5428	case SVM_EXIT_XCPT_SS:
5429	/* SVM_EXIT_XCPT_GP: / / Handled above. */
5430	/* SVM_EXIT_XCPT_PF: */
5431	case SVM_EXIT_XCPT_15: /* Reserved. */
5432	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5433	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5434	case SVM_EXIT_XCPT_MC:
5435	case SVM_EXIT_XCPT_XF:
5436	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5437	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5438	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5439	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5440
5441	case SVM_EXIT_SWINT: VMEXIT_CALL_RET(0, hmR0SvmExitSwInt(pVCpu, pSvmTransient));
5442	case SVM_EXIT_TR_READ: VMEXIT_CALL_RET(0, hmR0SvmExitTrRead(pVCpu, pSvmTransient));
5443	case SVM_EXIT_TR_WRITE: VMEXIT_CALL_RET(0, hmR0SvmExitTrWrite(pVCpu, pSvmTransient)); /* Also OS/2 TLB workaround. */
5444
5445	default:
5446	{
5447	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5448	pVCpu->hm.s.u32HMError = uExitCode;
5449	return VERR_SVM_UNKNOWN_EXIT;
5450	}
5451	}
5452	}
5453	}
5454	/* not reached */
5455	}
5456
5457
5458	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace probes.
5459	*
5460	* The following few functions and associated structure contains the bloat
5461	* necessary for providing detailed debug events and dtrace probes as well as
5462	* reliable host side single stepping. This works on the principle of
5463	* "subclassing" the normal execution loop and workers. We replace the loop
5464	* method completely and override selected helpers to add necessary adjustments
5465	* to their core operation.
5466	*
5467	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
5468	* any performance for debug and analysis features.
5469	*
5470	* @{
5471	*/
5472
5473	/**
5474	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
5475	* the debug run loop.
5476	*/
5477	typedef struct SVMRUNDBGSTATE
5478	{
5479	/** The initial SVMVMCBCTRL::u64InterceptCtrl value (helps with restore). */
5480	uint64_t bmInterceptInitial;
5481	/** The initial SVMVMCBCTRL::u32InterceptXcpt value (helps with restore). */
5482	uint32_t bmXcptInitial;
5483	/** The initial SVMVMCBCTRL::u16InterceptRdCRx value (helps with restore). */
5484	uint16_t bmInterceptRdCRxInitial;
5485	/** The initial SVMVMCBCTRL::u16InterceptWrCRx value (helps with restore). */
5486	uint16_t bmInterceptWrCRxInitial;
5487	/** The initial SVMVMCBCTRL::u16InterceptRdDRx value (helps with restore). */
5488	uint16_t bmInterceptRdDRxInitial;
5489	/** The initial SVMVMCBCTRL::u16InterceptWrDRx value (helps with restore). */
5490	uint16_t bmInterceptWrDRxInitial;
5491
5492	/** Whether we've actually modified the intercept control qword. */
5493	bool fModifiedInterceptCtrl : 1;
5494	/** Whether we've actually modified the exception bitmap. */
5495	bool fModifiedXcptBitmap : 1;
5496	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdCRx. */
5497	bool fModifiedInterceptRdCRx : 1;
5498	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrCRx. */
5499	bool fModifiedInterceptWrCRx : 1;
5500	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdDRx. */
5501	bool fModifiedInterceptRdDRx : 1;
5502	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrDRx. */
5503	bool fModifiedInterceptWrDRx : 1;
5504
5505	/** The CS we started executing with. */
5506	uint16_t uCsStart;
5507	/** The RIP we started executing at. This is for detecting that we stepped. */
5508	uint64_t uRipStart;
5509
5510	/** The sequence number of the Dtrace provider settings the state was
5511	* configured against. */
5512	uint32_t uDtraceSettingsSeqNo;
5513	/** Extra stuff we need in SVMVMCBCTRL::u32InterceptXcpt. */
5514	uint32_t bmXcptExtra;
5515	/** Extra stuff we need in SVMVMCBCTRL::u64InterceptCtrl. */
5516	uint64_t bmInterceptExtra;
5517	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdCRx. */
5518	uint16_t bmInterceptRdCRxExtra;
5519	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrCRx. */
5520	uint16_t bmInterceptWrCRxExtra;
5521	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdDRx. */
5522	uint16_t bmInterceptRdDRxExtra;
5523	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrDRx. */
5524	uint16_t bmInterceptWrDRxExtra;
5525	/** VM-exits to check (one bit per VM-exit). */
5526	uint32_t bmExitsToCheck[33];
5527	} SVMRUNDBGSTATE;
5528	AssertCompileMemberSize(SVMRUNDBGSTATE, bmExitsToCheck, (SVM_EXIT_MAX + 1 + 31) / 32 * 4);
5529	typedef SVMRUNDBGSTATE *PSVMRUNDBGSTATE;
5530
5531
5532	/**
5533	* Initializes the SVMRUNDBGSTATE structure.
5534	*
5535	* @param pVCpu The cross context virtual CPU structure of the
5536	* calling EMT.
5537	* @param pSvmTransient The SVM-transient structure.
5538	* @param pDbgState The debug state to initialize.
5539	*/
5540	static void hmR0SvmRunDebugStateInit(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5541	{
5542	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
5543	pDbgState->bmInterceptInitial = pVmcb->ctrl.u64InterceptCtrl;
5544	pDbgState->bmXcptInitial = pVmcb->ctrl.u32InterceptXcpt;
5545	pDbgState->bmInterceptRdCRxInitial = pVmcb->ctrl.u16InterceptRdCRx;
5546	pDbgState->bmInterceptWrCRxInitial = pVmcb->ctrl.u16InterceptWrCRx;
5547	pDbgState->bmInterceptRdDRxInitial = pVmcb->ctrl.u16InterceptRdDRx;
5548	pDbgState->bmInterceptWrDRxInitial = pVmcb->ctrl.u16InterceptWrDRx;
5549
5550	pDbgState->fModifiedInterceptCtrl = false;
5551	pDbgState->fModifiedXcptBitmap = false;
5552	pDbgState->fModifiedInterceptRdCRx = false;
5553	pDbgState->fModifiedInterceptWrCRx = false;
5554	pDbgState->fModifiedInterceptRdDRx = false;
5555	pDbgState->fModifiedInterceptWrDRx = false;
5556
5557	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
5558	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
5559
5560	/* We don't really need to zero these. */
5561	pDbgState->bmInterceptExtra = 0;
5562	pDbgState->bmXcptExtra = 0;
5563	pDbgState->bmInterceptRdCRxExtra = 0;
5564	pDbgState->bmInterceptWrCRxExtra = 0;
5565	pDbgState->bmInterceptRdDRxExtra = 0;
5566	pDbgState->bmInterceptWrDRxExtra = 0;
5567	}
5568
5569
5570	/**
5571	* Updates the VMCB fields with changes requested by @a pDbgState.
5572	*
5573	* This is performed after hmR0SvmPreRunGuestDebugStateUpdate as well
5574	* immediately before executing guest code, i.e. when interrupts are disabled.
5575	* We don't check status codes here as we cannot easily assert or return in the
5576	* latter case.
5577	*
5578	* @param pSvmTransient The SVM-transient structure.
5579	* @param pDbgState The debug state.
5580	*/
5581	static void hmR0SvmPreRunGuestDebugStateApply(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5582	{
5583	/*
5584	* Ensure desired flags in VMCS control fields are set.
5585	*/
5586	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5587	#define ADD_EXTRA_INTERCEPTS(a_VmcbCtrlField, a_bmExtra, a_fModified) do { \
5588	if ((pVmcb->ctrl. a_VmcbCtrlField & (a_bmExtra)) != (a_bmExtra)) \
5589	{ \
5590	pVmcb->ctrl. a_VmcbCtrlField \|= (a_bmExtra); \
5591	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5592	Log6Func((#a_VmcbCtrlField ": %#RX64\n", pVmcb->ctrl. a_VmcbCtrlField)); \
5593	(a_fModified) = true; \
5594	} \
5595	} while (0)
5596	ADD_EXTRA_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptExtra, pDbgState->fModifiedInterceptCtrl);
5597	ADD_EXTRA_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptExtra, pDbgState->fModifiedXcptBitmap);
5598	ADD_EXTRA_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxExtra, pDbgState->fModifiedInterceptRdCRx);
5599	ADD_EXTRA_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxExtra, pDbgState->fModifiedInterceptWrCRx);
5600	ADD_EXTRA_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxExtra, pDbgState->fModifiedInterceptRdDRx);
5601	ADD_EXTRA_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxExtra, pDbgState->fModifiedInterceptWrDRx);
5602	#undef ADD_EXTRA_INTERCEPTS
5603	}
5604
5605
5606	/**
5607	* Restores VMCB fields that were changed by hmR0SvmPreRunGuestDebugStateApply
5608	* for re-entry next time around.
5609	*
5610	* @param pSvmTransient The SVM-transient structure.
5611	* @param pDbgState The debug state.
5612	*/
5613	static void hmR0SvmRunDebugStateRevert(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5614	{
5615	/*
5616	* Restore VM-exit control settings as we may not reenter this function the
5617	* next time around.
5618	*/
5619	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5620
5621	#define RESTORE_INTERCEPTS(a_VmcbCtrlField, a_bmInitial, a_fModified) do { \
5622	if ((a_fModified)) \
5623	{ \
5624	pVmcb->ctrl. a_VmcbCtrlField = (a_bmInitial); \
5625	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5626	} \
5627	} while (0)
5628	RESTORE_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptInitial, pDbgState->fModifiedInterceptCtrl);
5629	RESTORE_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptInitial, pDbgState->fModifiedXcptBitmap);
5630	RESTORE_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxInitial, pDbgState->fModifiedInterceptRdCRx);
5631	RESTORE_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxInitial, pDbgState->fModifiedInterceptWrCRx);
5632	RESTORE_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxInitial, pDbgState->fModifiedInterceptRdDRx);
5633	RESTORE_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxInitial, pDbgState->fModifiedInterceptWrDRx);
5634	#undef RESTORE_INTERCEPTS
5635	}
5636
5637
5638	/**
5639	* Configures VM-exit controls for current DBGF and DTrace settings.
5640	*
5641	* This updates @a pDbgState and the VMCB execution control fields (in the debug
5642	* state) to reflect the necessary VM-exits demanded by DBGF and DTrace.
5643	*
5644	* @param pVCpu The cross context virtual CPU structure.
5645	* @param pSvmTransient The SVM-transient structure. May update
5646	* fUpdatedTscOffsettingAndPreemptTimer.
5647	* @param pDbgState The debug state.
5648	*/
5649	static void hmR0SvmPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5650	{
5651	/*
5652	* Take down the dtrace serial number so we can spot changes.
5653	*/
5654	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
5655	ASMCompilerBarrier();
5656
5657	/*
5658	* Clear data members that we'll be rebuilding here.
5659	*/
5660	pDbgState->bmXcptExtra = 0;
5661	pDbgState->bmInterceptExtra = 0;
5662	pDbgState->bmInterceptRdCRxExtra = 0;
5663	pDbgState->bmInterceptWrCRxExtra = 0;
5664	pDbgState->bmInterceptRdDRxExtra = 0;
5665	pDbgState->bmInterceptWrDRxExtra = 0;
5666	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
5667	pDbgState->bmExitsToCheck[i] = 0;
5668
5669	/*
5670	* Software interrupts (INT XXh)
5671	*/
5672	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5673	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
5674	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
5675	{
5676	pDbgState->bmInterceptExtra \|= SVM_CTRL_INTERCEPT_INTN;
5677	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SWINT);
5678	}
5679
5680	/*
5681	* INT3 breakpoints - triggered by #BP exceptions.
5682	*/
5683	if (pVM->dbgf.ro.cEnabledSwBreakpoints > 0)
5684	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
5685
5686	/*
5687	* Exception bitmap and XCPT events+probes.
5688	*/
5689	#define SET_XCPT(a_iXcpt) do { \
5690	pDbgState->bmXcptExtra \|= RT_BIT_32(a_iXcpt); \
5691	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_XCPT_0 + (a_iXcpt)); \
5692	} while (0)
5693
5694	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
5695	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
5696	SET_XCPT(iXcpt);
5697
5698	if (VBOXVMM_XCPT_DE_ENABLED()) SET_XCPT(X86_XCPT_DE);
5699	if (VBOXVMM_XCPT_DB_ENABLED()) SET_XCPT(X86_XCPT_DB);
5700	if (VBOXVMM_XCPT_BP_ENABLED()) SET_XCPT(X86_XCPT_BP);
5701	if (VBOXVMM_XCPT_OF_ENABLED()) SET_XCPT(X86_XCPT_OF);
5702	if (VBOXVMM_XCPT_BR_ENABLED()) SET_XCPT(X86_XCPT_BR);
5703	if (VBOXVMM_XCPT_UD_ENABLED()) SET_XCPT(X86_XCPT_UD);
5704	if (VBOXVMM_XCPT_NM_ENABLED()) SET_XCPT(X86_XCPT_NM);
5705	if (VBOXVMM_XCPT_DF_ENABLED()) SET_XCPT(X86_XCPT_DF);
5706	if (VBOXVMM_XCPT_TS_ENABLED()) SET_XCPT(X86_XCPT_TS);
5707	if (VBOXVMM_XCPT_NP_ENABLED()) SET_XCPT(X86_XCPT_NP);
5708	if (VBOXVMM_XCPT_SS_ENABLED()) SET_XCPT(X86_XCPT_SS);
5709	if (VBOXVMM_XCPT_GP_ENABLED()) SET_XCPT(X86_XCPT_GP);
5710	if (VBOXVMM_XCPT_PF_ENABLED()) SET_XCPT(X86_XCPT_PF);
5711	if (VBOXVMM_XCPT_MF_ENABLED()) SET_XCPT(X86_XCPT_MF);
5712	if (VBOXVMM_XCPT_AC_ENABLED()) SET_XCPT(X86_XCPT_AC);
5713	if (VBOXVMM_XCPT_XF_ENABLED()) SET_XCPT(X86_XCPT_XF);
5714	if (VBOXVMM_XCPT_VE_ENABLED()) SET_XCPT(X86_XCPT_VE);
5715	if (VBOXVMM_XCPT_SX_ENABLED()) SET_XCPT(X86_XCPT_SX);
5716
5717	#undef SET_XCPT
5718
5719	/*
5720	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
5721	*
5722	* Note! This is the reverse of what hmR0SvmHandleExitDtraceEvents does.
5723	* So, when adding/changing/removing please don't forget to update it.
5724	*
5725	* Some of the macros are picking up local variables to save horizontal space,
5726	* (being able to see it in a table is the lesser evil here).
5727	*/
5728	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
5729	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
5730	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
5731	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
5732	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5733	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5734	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5735	} else do { } while (0)
5736	#define SET_INCP_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fInterceptCtrl) \
5737	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5738	{ \
5739	(pDbgState)->bmInterceptExtra \|= (a_fInterceptCtrl); \
5740	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5741	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5742	} else do { } while (0)
5743
5744	/** @todo double check these */
5745	/** @todo Check what more AMD-V specific we can intercept. */
5746	//SET_INCP_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH, SVM_CTRL_INTERCEPT_TASK_SWITCH);
5747	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH);
5748	SET_INCP_XBM_IF_EITHER_EN(INSTR_VMM_CALL, SVM_EXIT_VMMCALL, SVM_CTRL_INTERCEPT_VMMCALL);
5749	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, SVM_EXIT_VMMCALL);
5750	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMRUN, SVM_EXIT_VMRUN, SVM_CTRL_INTERCEPT_VMRUN);
5751	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMRUN, SVM_EXIT_VMRUN);
5752	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMLOAD, SVM_EXIT_VMLOAD, SVM_CTRL_INTERCEPT_VMLOAD);
5753	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMLOAD, SVM_EXIT_VMLOAD);
5754	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMSAVE, SVM_EXIT_VMSAVE, SVM_CTRL_INTERCEPT_VMSAVE);
5755	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMSAVE, SVM_EXIT_VMSAVE);
5756	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_STGI, SVM_EXIT_STGI, SVM_CTRL_INTERCEPT_STGI);
5757	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_STGI, SVM_EXIT_STGI);
5758	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_CLGI, SVM_EXIT_CLGI, SVM_CTRL_INTERCEPT_CLGI);
5759	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_CLGI, SVM_EXIT_CLGI);
5760
5761	SET_INCP_XBM_IF_EITHER_EN(INSTR_CPUID, SVM_EXIT_CPUID, SVM_CTRL_INTERCEPT_CPUID);
5762	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, SVM_EXIT_CPUID);
5763	SET_INCP_XBM_IF_EITHER_EN(INSTR_HALT, SVM_EXIT_HLT, SVM_CTRL_INTERCEPT_HLT);
5764	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, SVM_EXIT_HLT);
5765	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVD, SVM_EXIT_INVD, SVM_CTRL_INTERCEPT_INVD);
5766	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, SVM_EXIT_INVD);
5767	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVLPG, SVM_EXIT_INVLPG, SVM_CTRL_INTERCEPT_INVLPG);
5768	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, SVM_EXIT_INVLPG);
5769	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDPMC, SVM_EXIT_RDPMC, SVM_CTRL_INTERCEPT_RDPMC);
5770	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, SVM_EXIT_RDPMC);
5771	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSC, SVM_EXIT_RDTSC, SVM_CTRL_INTERCEPT_RDTSC);
5772	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, SVM_EXIT_RDTSC);
5773	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSCP, SVM_EXIT_RDTSCP, SVM_CTRL_INTERCEPT_RDTSCP);
5774	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, SVM_EXIT_RDTSCP);
5775	SET_INCP_XBM_IF_EITHER_EN(INSTR_RSM, SVM_EXIT_RSM, SVM_CTRL_INTERCEPT_RSM);
5776	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, SVM_EXIT_RSM);
5777
5778	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
5779	pDbgState->bmInterceptRdCRxExtra = 0xffff;
5780	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_READ))
5781	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_CR0, SVM_EXIT_READ_CR15 + 1);
5782
5783	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
5784	pDbgState->bmInterceptWrCRxExtra = 0xffff;
5785	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_WRITE))
5786	{
5787	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_CR0, SVM_EXIT_WRITE_CR15 + 1);
5788	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_CR0_SEL_WRITE);
5789	}
5790
5791	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ))
5792	pDbgState->bmInterceptRdDRxExtra = 0xffff;
5793	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_READ))
5794	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_DR0, SVM_EXIT_READ_DR15 + 1);
5795
5796	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
5797	pDbgState->bmInterceptWrDRxExtra = 0xffff;
5798	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_WRITE))
5799	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_DR0, SVM_EXIT_WRITE_DR15 + 1);
5800
5801	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RDMSR, SVM_EXIT_MSR); /** @todo modify bitmap to intercept almost everything? (Clearing MSR_PROT just means no intercepts.) */
5802	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, SVM_EXIT_MSR);
5803	SET_ONLY_XBM_IF_EITHER_EN(INSTR_WRMSR, SVM_EXIT_MSR); /** @todo ditto */
5804	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, SVM_EXIT_MSR);
5805	SET_INCP_XBM_IF_EITHER_EN(INSTR_MWAIT, SVM_EXIT_MWAIT, SVM_CTRL_INTERCEPT_MWAIT);
5806	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, SVM_EXIT_MWAIT);
5807	if (ASMBitTest(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT))
5808	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT_ARMED);
5809	SET_INCP_XBM_IF_EITHER_EN(INSTR_MONITOR, SVM_EXIT_MONITOR, SVM_CTRL_INTERCEPT_MONITOR);
5810	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, SVM_EXIT_MONITOR);
5811	SET_INCP_XBM_IF_EITHER_EN(INSTR_PAUSE, SVM_EXIT_PAUSE, SVM_CTRL_INTERCEPT_PAUSE);
5812	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, SVM_EXIT_PAUSE);
5813	SET_INCP_XBM_IF_EITHER_EN(INSTR_SIDT, SVM_EXIT_IDTR_READ, SVM_CTRL_INTERCEPT_IDTR_READS);
5814	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, SVM_EXIT_IDTR_READ);
5815	SET_INCP_XBM_IF_EITHER_EN(INSTR_LIDT, SVM_EXIT_IDTR_WRITE, SVM_CTRL_INTERCEPT_IDTR_WRITES);
5816	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, SVM_EXIT_IDTR_WRITE);
5817	SET_INCP_XBM_IF_EITHER_EN(INSTR_SGDT, SVM_EXIT_GDTR_READ, SVM_CTRL_INTERCEPT_GDTR_READS);
5818	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, SVM_EXIT_GDTR_READ);
5819	SET_INCP_XBM_IF_EITHER_EN(INSTR_LGDT, SVM_EXIT_GDTR_WRITE, SVM_CTRL_INTERCEPT_GDTR_WRITES);
5820	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, SVM_EXIT_GDTR_WRITE);
5821	SET_INCP_XBM_IF_EITHER_EN(INSTR_SLDT, SVM_EXIT_LDTR_READ, SVM_CTRL_INTERCEPT_LDTR_READS);
5822	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, SVM_EXIT_LDTR_READ);
5823	SET_INCP_XBM_IF_EITHER_EN(INSTR_LLDT, SVM_EXIT_LDTR_WRITE, SVM_CTRL_INTERCEPT_LDTR_WRITES);
5824	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, SVM_EXIT_LDTR_WRITE);
5825	SET_INCP_XBM_IF_EITHER_EN(INSTR_STR, SVM_EXIT_TR_READ, SVM_CTRL_INTERCEPT_TR_READS);
5826	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, SVM_EXIT_TR_READ);
5827	SET_INCP_XBM_IF_EITHER_EN(INSTR_LTR, SVM_EXIT_TR_WRITE, SVM_CTRL_INTERCEPT_TR_WRITES);
5828	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, SVM_EXIT_TR_WRITE);
5829	SET_INCP_XBM_IF_EITHER_EN(INSTR_WBINVD, SVM_EXIT_WBINVD, SVM_CTRL_INTERCEPT_WBINVD);
5830	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, SVM_EXIT_WBINVD);
5831	SET_INCP_XBM_IF_EITHER_EN(INSTR_XSETBV, SVM_EXIT_XSETBV, SVM_CTRL_INTERCEPT_XSETBV);
5832	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, SVM_EXIT_XSETBV);
5833
5834	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_TRIPLE_FAULT))
5835	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SHUTDOWN);
5836
5837	#undef IS_EITHER_ENABLED
5838	#undef SET_ONLY_XBM_IF_EITHER_EN
5839	#undef SET_INCP_XBM_IF_EITHER_EN
5840
5841	/*
5842	* Sanitize the control stuff.
5843	*/
5844	/** @todo filter out unsupported stuff? */
5845	if ( pVCpu->hmr0.s.fDebugWantRdTscExit
5846	!= RT_BOOL(pDbgState->bmInterceptExtra & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
5847	{
5848	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
5849	/// @todo pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
5850	RT_NOREF(pSvmTransient);
5851	}
5852
5853	Log6(("HM: debug state: bmInterceptExtra=%#RX64 bmXcptExtra=%#RX32%s%s%s%s bmExitsToCheck=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
5854	pDbgState->bmInterceptExtra, pDbgState->bmXcptExtra,
5855	pDbgState->bmInterceptRdCRxExtra ? " rd-cr" : "",
5856	pDbgState->bmInterceptWrCRxExtra ? " wr-cr" : "",
5857	pDbgState->bmInterceptRdDRxExtra ? " rd-dr" : "",
5858	pDbgState->bmInterceptWrDRxExtra ? " wr-dr" : "",
5859	pDbgState->bmExitsToCheck[0],
5860	pDbgState->bmExitsToCheck[1],
5861	pDbgState->bmExitsToCheck[2],
5862	pDbgState->bmExitsToCheck[3],
5863	pDbgState->bmExitsToCheck[4]));
5864	}
5865
5866
5867	/**
5868	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
5869	* appropriate.
5870	*
5871	* The caller has checked the VM-exit against the SVMRUNDBGSTATE::bmExitsToCheck
5872	* bitmap.
5873	*
5874	* @returns Strict VBox status code (i.e. informational status codes too).
5875	* @param pVCpu The cross context virtual CPU structure.
5876	* @param pSvmTransient The SVM-transient structure.
5877	* @param uExitCode The VM-exit code.
5878	*
5879	* @remarks The name of this function is displayed by dtrace, so keep it short
5880	* and to the point. No longer than 33 chars long, please.
5881	*/
5882	static VBOXSTRICTRC hmR0SvmHandleExitDtraceEvents(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, uint64_t uExitCode)
5883	{
5884	/*
5885	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
5886	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
5887	*
5888	* Note! This is the reverse operation of what hmR0SvmPreRunGuestDebugStateUpdate
5889	* does. Must add/change/remove both places. Same ordering, please.
5890	*
5891	* Added/removed events must also be reflected in the next section
5892	* where we dispatch dtrace events.
5893	*/
5894	bool fDtrace1 = false;
5895	bool fDtrace2 = false;
5896	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
5897	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
5898	uint64_t uEventArg = 0;
5899	#define SET_XCPT(a_XcptName) \
5900	do { \
5901	enmEvent2 = RT_CONCAT(DBGFEVENT_XCPT_, a_XcptName); \
5902	fDtrace2 = RT_CONCAT3(VBOXVMM_XCPT_, a_XcptName, _ENABLED)(); \
5903	} while (0)
5904	#define SET_EXIT(a_EventSubName) \
5905	do { \
5906	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5907	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5908	} while (0)
5909	#define SET_BOTH(a_EventSubName) \
5910	do { \
5911	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
5912	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5913	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
5914	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5915	} while (0)
5916	switch (uExitCode)
5917	{
5918	case SVM_EXIT_SWINT:
5919	enmEvent2 = DBGFEVENT_INTERRUPT_SOFTWARE;
5920	fDtrace2 = VBOXVMM_INT_SOFTWARE_ENABLED();
5921	uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1;
5922	break;
5923
5924	case SVM_EXIT_XCPT_DE: SET_XCPT(DE); break;
5925	case SVM_EXIT_XCPT_DB: SET_XCPT(DB); break;
5926	case SVM_EXIT_XCPT_BP: SET_XCPT(BP); break;
5927	case SVM_EXIT_XCPT_OF: SET_XCPT(OF); break;
5928	case SVM_EXIT_XCPT_BR: SET_XCPT(BR); break;
5929	case SVM_EXIT_XCPT_UD: SET_XCPT(UD); break;
5930	case SVM_EXIT_XCPT_NM: SET_XCPT(NM); break;
5931	case SVM_EXIT_XCPT_DF: SET_XCPT(DF); break;
5932	case SVM_EXIT_XCPT_TS: SET_XCPT(TS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5933	case SVM_EXIT_XCPT_NP: SET_XCPT(NP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5934	case SVM_EXIT_XCPT_SS: SET_XCPT(SS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5935	case SVM_EXIT_XCPT_GP: SET_XCPT(GP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5936	case SVM_EXIT_XCPT_PF: SET_XCPT(PF); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5937	case SVM_EXIT_XCPT_MF: SET_XCPT(MF); break;
5938	case SVM_EXIT_XCPT_AC: SET_XCPT(AC); break;
5939	case SVM_EXIT_XCPT_XF: SET_XCPT(XF); break;
5940	case SVM_EXIT_XCPT_VE: SET_XCPT(VE); break;
5941	case SVM_EXIT_XCPT_SX: SET_XCPT(SX); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5942
5943	case SVM_EXIT_XCPT_2: enmEvent2 = DBGFEVENT_XCPT_02; break;
5944	case SVM_EXIT_XCPT_9: enmEvent2 = DBGFEVENT_XCPT_09; break;
5945	case SVM_EXIT_XCPT_15: enmEvent2 = DBGFEVENT_XCPT_0f; break;
5946	case SVM_EXIT_XCPT_18: enmEvent2 = DBGFEVENT_XCPT_MC; break;
5947	case SVM_EXIT_XCPT_21: enmEvent2 = DBGFEVENT_XCPT_15; break;
5948	case SVM_EXIT_XCPT_22: enmEvent2 = DBGFEVENT_XCPT_16; break;
5949	case SVM_EXIT_XCPT_23: enmEvent2 = DBGFEVENT_XCPT_17; break;
5950	case SVM_EXIT_XCPT_24: enmEvent2 = DBGFEVENT_XCPT_18; break;
5951	case SVM_EXIT_XCPT_25: enmEvent2 = DBGFEVENT_XCPT_19; break;
5952	case SVM_EXIT_XCPT_26: enmEvent2 = DBGFEVENT_XCPT_1a; break;
5953	case SVM_EXIT_XCPT_27: enmEvent2 = DBGFEVENT_XCPT_1b; break;
5954	case SVM_EXIT_XCPT_28: enmEvent2 = DBGFEVENT_XCPT_1c; break;
5955	case SVM_EXIT_XCPT_29: enmEvent2 = DBGFEVENT_XCPT_1d; break;
5956	case SVM_EXIT_XCPT_31: enmEvent2 = DBGFEVENT_XCPT_1f; break;
5957
5958	case SVM_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
5959	case SVM_EXIT_VMMCALL: SET_BOTH(VMM_CALL); break;
5960	case SVM_EXIT_VMRUN: SET_BOTH(SVM_VMRUN); break;
5961	case SVM_EXIT_VMLOAD: SET_BOTH(SVM_VMLOAD); break;
5962	case SVM_EXIT_VMSAVE: SET_BOTH(SVM_VMSAVE); break;
5963	case SVM_EXIT_STGI: SET_BOTH(SVM_STGI); break;
5964	case SVM_EXIT_CLGI: SET_BOTH(SVM_CLGI); break;
5965	case SVM_EXIT_CPUID: SET_BOTH(CPUID); break;
5966	case SVM_EXIT_HLT: SET_BOTH(HALT); break;
5967	case SVM_EXIT_INVD: SET_BOTH(INVD); break;
5968	case SVM_EXIT_INVLPG: SET_BOTH(INVLPG); break;
5969	case SVM_EXIT_RDPMC: SET_BOTH(RDPMC); break;
5970	case SVM_EXIT_RDTSC: SET_BOTH(RDTSC); break;
5971	case SVM_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
5972	case SVM_EXIT_RSM: SET_BOTH(RSM); break;
5973
5974	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
5975	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
5976	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
5977	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
5978	SET_BOTH(CRX_READ);
5979	uEventArg = uExitCode - SVM_EXIT_READ_CR0;
5980	break;
5981	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
5982	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
5983	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
5984	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
5985	case SVM_EXIT_CR0_SEL_WRITE:
5986	SET_BOTH(CRX_WRITE);
5987	uEventArg = uExitCode - SVM_EXIT_WRITE_CR0;
5988	break;
5989	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5990	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
5991	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
5992	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5993	SET_BOTH(DRX_READ);
5994	uEventArg = uExitCode - SVM_EXIT_READ_DR0;
5995	break;
5996	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5997	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
5998	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
5999	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6000	SET_BOTH(DRX_WRITE);
6001	uEventArg = uExitCode - SVM_EXIT_WRITE_DR0;
6002	break;
6003	case SVM_EXIT_MSR:
6004	if (pSvmTransient->pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6005	SET_BOTH(WRMSR);
6006	else
6007	SET_BOTH(RDMSR);
6008	break;
6009	case SVM_EXIT_MWAIT_ARMED:
6010	case SVM_EXIT_MWAIT: SET_BOTH(MWAIT); break;
6011	case SVM_EXIT_MONITOR: SET_BOTH(MONITOR); break;
6012	case SVM_EXIT_PAUSE: SET_BOTH(PAUSE); break;
6013	case SVM_EXIT_IDTR_READ: SET_BOTH(SIDT); break;
6014	case SVM_EXIT_IDTR_WRITE: SET_BOTH(LIDT); break;
6015	case SVM_EXIT_GDTR_READ: SET_BOTH(SGDT); break;
6016	case SVM_EXIT_GDTR_WRITE: SET_BOTH(LGDT); break;
6017	case SVM_EXIT_LDTR_READ: SET_BOTH(SLDT); break;
6018	case SVM_EXIT_LDTR_WRITE: SET_BOTH(LLDT); break;
6019	case SVM_EXIT_TR_READ: SET_BOTH(STR); break;
6020	case SVM_EXIT_TR_WRITE: SET_BOTH(LTR); break;
6021	case SVM_EXIT_WBINVD: SET_BOTH(WBINVD); break;
6022	case SVM_EXIT_XSETBV: SET_BOTH(XSETBV); break;
6023
6024	case SVM_EXIT_SHUTDOWN:
6025	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
6026	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
6027	break;
6028
6029	default:
6030	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6031	break;
6032	}
6033	#undef SET_BOTH
6034	#undef SET_EXIT
6035
6036	/*
6037	* Dtrace tracepoints go first. We do them here at once so we don't
6038	* have to copy the guest state saving and stuff a few dozen times.
6039	* Down side is that we've got to repeat the switch, though this time
6040	* we use enmEvent since the probes are a subset of what DBGF does.
6041	*/
6042	if (fDtrace1 \|\| fDtrace2)
6043	{
6044	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6045	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; RT_NOREF(pCtx); /* Shut up Clang 13. */
6046	switch (enmEvent1)
6047	{
6048	/** @todo consider which extra parameters would be helpful for each probe. */
6049	case DBGFEVENT_END: break;
6050	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6051	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
6052	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
6053	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
6054	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
6055	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
6056	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
6057	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
6058	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
6059	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6060	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6061	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6062	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6063	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, (uint32_t)uEventArg, pCtx->cr2); break;
6064	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
6065	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
6066	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
6067	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
6068	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, (uint32_t)uEventArg); break;
6069	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6070	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
6071	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
6072	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
6073	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
6074	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
6075	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
6076	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6077	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6078	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6079	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6080	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6081	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
6082	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6083	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
6084	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
6085	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
6086	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
6087	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
6088	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
6089	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
6090	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
6091	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
6092	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
6093	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
6094	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
6095	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
6096	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
6097	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
6098	case DBGFEVENT_INSTR_SVM_VMRUN: VBOXVMM_INSTR_SVM_VMRUN(pVCpu, pCtx); break;
6099	case DBGFEVENT_INSTR_SVM_VMLOAD: VBOXVMM_INSTR_SVM_VMLOAD(pVCpu, pCtx); break;
6100	case DBGFEVENT_INSTR_SVM_VMSAVE: VBOXVMM_INSTR_SVM_VMSAVE(pVCpu, pCtx); break;
6101	case DBGFEVENT_INSTR_SVM_STGI: VBOXVMM_INSTR_SVM_STGI(pVCpu, pCtx); break;
6102	case DBGFEVENT_INSTR_SVM_CLGI: VBOXVMM_INSTR_SVM_CLGI(pVCpu, pCtx); break;
6103	default: AssertMsgFailed(("enmEvent1=%d uExitCode=%d\n", enmEvent1, uExitCode)); break;
6104	}
6105	switch (enmEvent2)
6106	{
6107	/** @todo consider which extra parameters would be helpful for each probe. */
6108	case DBGFEVENT_END: break;
6109	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
6110	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6111	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
6112	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
6113	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
6114	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
6115	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
6116	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
6117	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6118	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6119	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6120	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6121	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6122	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
6123	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6124	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
6125	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
6126	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
6127	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
6128	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
6129	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
6130	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
6131	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
6132	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
6133	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
6134	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
6135	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
6136	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
6137	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
6138	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
6139	case DBGFEVENT_EXIT_SVM_VMRUN: VBOXVMM_EXIT_SVM_VMRUN(pVCpu, pCtx); break;
6140	case DBGFEVENT_EXIT_SVM_VMLOAD: VBOXVMM_EXIT_SVM_VMLOAD(pVCpu, pCtx); break;
6141	case DBGFEVENT_EXIT_SVM_VMSAVE: VBOXVMM_EXIT_SVM_VMSAVE(pVCpu, pCtx); break;
6142	case DBGFEVENT_EXIT_SVM_STGI: VBOXVMM_EXIT_SVM_STGI(pVCpu, pCtx); break;
6143	case DBGFEVENT_EXIT_SVM_CLGI: VBOXVMM_EXIT_SVM_CLGI(pVCpu, pCtx); break;
6144	default: AssertMsgFailed(("enmEvent2=%d uExitCode=%d\n", enmEvent2, uExitCode)); break;
6145	}
6146	}
6147
6148	/*
6149	* Fire of the DBGF event, if enabled (our check here is just a quick one,
6150	* the DBGF call will do a full check).
6151	*
6152	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
6153	* Note! If we have to events, we prioritize the first, i.e. the instruction
6154	* one, in order to avoid event nesting.
6155	*/
6156	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6157	VBOXSTRICTRC rcStrict;
6158	if ( enmEvent1 != DBGFEVENT_END
6159	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
6160	{
6161	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6162	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
6163	}
6164	else if ( enmEvent2 != DBGFEVENT_END
6165	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
6166	{
6167	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6168	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
6169	}
6170	else
6171	rcStrict = VINF_SUCCESS;
6172	return rcStrict;
6173	}
6174
6175
6176	/**
6177	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID),
6178	* debug variant.
6179	*
6180	* @returns Strict VBox status code (informational status codes included).
6181	* @param pVCpu The cross context virtual CPU structure.
6182	* @param pSvmTransient Pointer to the SVM transient structure.
6183	* @param pDbgState The runtime debug state.
6184	*/
6185	static VBOXSTRICTRC hmR0SvmDebugHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
6186	{
6187	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
6188	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
6189
6190	/*
6191	* Expensive (saves context) generic dtrace VM-exit probe.
6192	*/
6193	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6194	if (!VBOXVMM_R0_HMSVM_VMEXIT_ENABLED())
6195	{ /* more likely */ }
6196	else
6197	{
6198	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6199	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, uExitCode, pSvmTransient->pVmcb);
6200	}
6201
6202	/*
6203	* Check for single stepping event if we're stepping.
6204	*/
6205	if (pVCpu->hm.s.fSingleInstruction)
6206	{
6207	switch (uExitCode)
6208	{
6209	/* Various events: */
6210	case SVM_EXIT_XCPT_0: case SVM_EXIT_XCPT_1: case SVM_EXIT_XCPT_2: case SVM_EXIT_XCPT_3:
6211	case SVM_EXIT_XCPT_4: case SVM_EXIT_XCPT_5: case SVM_EXIT_XCPT_6: case SVM_EXIT_XCPT_7:
6212	case SVM_EXIT_XCPT_8: case SVM_EXIT_XCPT_9: case SVM_EXIT_XCPT_10: case SVM_EXIT_XCPT_11:
6213	case SVM_EXIT_XCPT_12: case SVM_EXIT_XCPT_13: case SVM_EXIT_XCPT_14: case SVM_EXIT_XCPT_15:
6214	case SVM_EXIT_XCPT_16: case SVM_EXIT_XCPT_17: case SVM_EXIT_XCPT_18: case SVM_EXIT_XCPT_19:
6215	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
6216	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
6217	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
6218	case SVM_EXIT_INTR:
6219	case SVM_EXIT_NMI:
6220	case SVM_EXIT_VINTR:
6221	case SVM_EXIT_NPF:
6222	case SVM_EXIT_AVIC_NOACCEL:
6223
6224	/* Instruction specific VM-exits: */
6225	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6226	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6227	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6228	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6229	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6230	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6231	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6232	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6233	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6234	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6235	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6236	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6237	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6238	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6239	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6240	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6241	case SVM_EXIT_CR0_SEL_WRITE:
6242	case SVM_EXIT_IDTR_READ:
6243	case SVM_EXIT_GDTR_READ:
6244	case SVM_EXIT_LDTR_READ:
6245	case SVM_EXIT_TR_READ:
6246	case SVM_EXIT_IDTR_WRITE:
6247	case SVM_EXIT_GDTR_WRITE:
6248	case SVM_EXIT_LDTR_WRITE:
6249	case SVM_EXIT_TR_WRITE:
6250	case SVM_EXIT_RDTSC:
6251	case SVM_EXIT_RDPMC:
6252	case SVM_EXIT_PUSHF:
6253	case SVM_EXIT_POPF:
6254	case SVM_EXIT_CPUID:
6255	case SVM_EXIT_RSM:
6256	case SVM_EXIT_IRET:
6257	case SVM_EXIT_SWINT:
6258	case SVM_EXIT_INVD:
6259	case SVM_EXIT_PAUSE:
6260	case SVM_EXIT_HLT:
6261	case SVM_EXIT_INVLPG:
6262	case SVM_EXIT_INVLPGA:
6263	case SVM_EXIT_IOIO:
6264	case SVM_EXIT_MSR:
6265	case SVM_EXIT_TASK_SWITCH:
6266	case SVM_EXIT_VMRUN:
6267	case SVM_EXIT_VMMCALL:
6268	case SVM_EXIT_VMLOAD:
6269	case SVM_EXIT_VMSAVE:
6270	case SVM_EXIT_STGI:
6271	case SVM_EXIT_CLGI:
6272	case SVM_EXIT_SKINIT:
6273	case SVM_EXIT_RDTSCP:
6274	case SVM_EXIT_ICEBP:
6275	case SVM_EXIT_WBINVD:
6276	case SVM_EXIT_MONITOR:
6277	case SVM_EXIT_MWAIT:
6278	case SVM_EXIT_MWAIT_ARMED:
6279	case SVM_EXIT_XSETBV:
6280	case SVM_EXIT_RDPRU:
6281	case SVM_EXIT_WRITE_EFER_TRAP:
6282	case SVM_EXIT_WRITE_CR0_TRAP: case SVM_EXIT_WRITE_CR1_TRAP: case SVM_EXIT_WRITE_CR2_TRAP: case SVM_EXIT_WRITE_CR3_TRAP:
6283	case SVM_EXIT_WRITE_CR4_TRAP: case SVM_EXIT_WRITE_CR5_TRAP: case SVM_EXIT_WRITE_CR6_TRAP: case SVM_EXIT_WRITE_CR7_TRAP:
6284	case SVM_EXIT_WRITE_CR8_TRAP: case SVM_EXIT_WRITE_CR9_TRAP: case SVM_EXIT_WRITE_CR10_TRAP: case SVM_EXIT_WRITE_CR11_TRAP:
6285	case SVM_EXIT_WRITE_CR12_TRAP: case SVM_EXIT_WRITE_CR13_TRAP: case SVM_EXIT_WRITE_CR14_TRAP: case SVM_EXIT_WRITE_CR15_TRAP:
6286	case SVM_EXIT_MCOMMIT:
6287	{
6288	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6289	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
6290	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
6291	{
6292	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6293	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode));
6294	return VINF_EM_DBG_STEPPED;
6295	}
6296	break;
6297	}
6298
6299	/* Errors and unexpected events: */
6300	case SVM_EXIT_FERR_FREEZE:
6301	case SVM_EXIT_SHUTDOWN:
6302	case SVM_EXIT_AVIC_INCOMPLETE_IPI:
6303	break;
6304
6305	case SVM_EXIT_SMI:
6306	case SVM_EXIT_INIT:
6307	default:
6308	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6309	break;
6310	}
6311	}
6312
6313	/*
6314	* Check for debugger event breakpoints and dtrace probes.
6315	*/
6316	if ( uExitCode < sizeof(pDbgState->bmExitsToCheck) * 8U
6317	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitCode) )
6318	{
6319	VBOXSTRICTRC rcStrict = hmR0SvmHandleExitDtraceEvents(pVCpu, pSvmTransient, uExitCode);
6320	if (rcStrict != VINF_SUCCESS)
6321	{
6322	Log6Func(("%04x:%08RX64 (exit %u) -> %Rrc\n",
6323	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, VBOXSTRICTRC_VAL(rcStrict) ));
6324	return rcStrict;
6325	}
6326	}
6327
6328	/*
6329	* Normal processing.
6330	*/
6331	return hmR0SvmHandleExit(pVCpu, pSvmTransient);
6332	}
6333
6334
6335	/**
6336	* Runs the guest code using AMD-V in single step mode.
6337	*
6338	* @returns Strict VBox status code.
6339	* @param pVCpu The cross context virtual CPU structure.
6340	* @param pcLoops Pointer to the number of executed loops.
6341	*/
6342	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
6343	{
6344	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6345	Assert(pcLoops);
6346	Assert(*pcLoops <= cMaxResumeLoops);
6347
6348	SVMTRANSIENT SvmTransient;
6349	RT_ZERO(SvmTransient);
6350	SvmTransient.fUpdateTscOffsetting = true;
6351	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6352
6353	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6354
6355	/* Set HMCPU indicators. */
6356	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
6357	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
6358	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6359	pVCpu->hmr0.s.fUsingDebugLoop = true;
6360
6361	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
6362	SVMRUNDBGSTATE DbgState;
6363	hmR0SvmRunDebugStateInit(pVCpu, &SvmTransient, &DbgState);
6364	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6365
6366	/*
6367	* The loop.
6368	*/
6369	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
6370	for (;;)
6371	{
6372	Assert(!HMR0SuspendPending());
6373	AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
6374	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
6375	(unsigned)RTMpCpuId(), *pcLoops));
6376	bool fStepping = pVCpu->hm.s.fSingleInstruction;
6377
6378	/* Set up VM-execution controls the next two can respond to. */
6379	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6380
6381	/* Preparatory work for running nested-guest code, this may force us to return to
6382	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
6383	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6384	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
6385	if (rc != VINF_SUCCESS)
6386	break;
6387
6388	/*
6389	* No longjmps to ring-3 from this point on!!!
6390	*
6391	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
6392	* better than a kernel panic. This also disables flushing of the R0-logger instance.
6393	*/
6394	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
6395
6396	/* Override any obnoxious code in the above two calls. */
6397	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6398	#if 0
6399	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64\n",
6400	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6401	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u));
6402	#endif
6403
6404	/*
6405	* Finally execute guest code.
6406	*/
6407	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
6408
6409	/* Restore any residual host-state and save any bits shared between host and guest
6410	into the guest-CPU state. Re-enables interrupts! */
6411	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
6412	#if 0
6413	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64 exit=%d\n",
6414	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6415	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u, SvmTransient.u64ExitCode));
6416	#endif
6417
6418	if (RT_LIKELY( rc == VINF_SUCCESS /* Check for VMRUN errors. */
6419	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
6420	{ /* very likely*/ }
6421	else
6422	{
6423	if (rc == VINF_SUCCESS)
6424	rc = VERR_SVM_INVALID_GUEST_STATE;
6425	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6426	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
6427	return rc;
6428	}
6429
6430	/* Handle the #VMEXIT. */
6431	HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
6432	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6433	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
6434	rc = hmR0SvmDebugHandleExit(pVCpu, &SvmTransient, &DbgState);
6435	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6436	if (rc != VINF_SUCCESS)
6437	break;
6438	if (++(*pcLoops) >= cMaxResumeLoops)
6439	{
6440	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6441	rc = VINF_EM_RAW_INTERRUPT;
6442	break;
6443	}
6444
6445	/*
6446	* Stepping: Did the RIP change, if so, consider it a single step.
6447	* Otherwise, make sure one of the TFs gets set.
6448	*/
6449	if (fStepping)
6450	{
6451	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6452	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
6453	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
6454	{
6455	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6456	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, SvmTransient.u64ExitCode));
6457	rc = VINF_EM_DBG_STEPPED;
6458	break;
6459	}
6460	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
6461	}
6462
6463	/*
6464	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
6465	* Revert the state changes afterware so we can drop intercepts no longer needed.
6466	*/
6467	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
6468	{
6469	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6470	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6471	}
6472	}
6473
6474	/*
6475	* Clear the X86_EFL_TF if necessary.
6476	*/
6477	if (pVCpu->hmr0.s.fClearTrapFlag)
6478	{
6479	pVCpu->hmr0.s.fClearTrapFlag = false;
6480	pCtx->eflags.Bits.u1TF = 0;
6481	}
6482
6483	/* Restore HMCPU indicators. */
6484	pVCpu->hmr0.s.fUsingDebugLoop = false;
6485	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6486	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
6487
6488	/* Restore all controls applied by hmR0SvmPreRunGuestDebugStateApply above. */
6489	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6490
6491	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6492	return rc;
6493	}
6494
6495	/** @} */
6496
6497	#undef VMEXIT_CALL_RET
6498
6499
6500	#ifdef VBOX_STRICT
6501	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6502	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
6503	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6504
6505	# define HMSVM_ASSERT_PREEMPT_CPUID() \
6506	do \
6507	{ \
6508	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6509	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6510	} while (0)
6511
6512	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6513	do { \
6514	AssertPtr((a_pVCpu)); \
6515	AssertPtr((a_pSvmTransient)); \
6516	Assert(ASMIntAreEnabled()); \
6517	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6518	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
6519	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
6520	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6521	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6522	HMSVM_ASSERT_PREEMPT_CPUID(); \
6523	} while (0)
6524	#else
6525	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6526	do { \
6527	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
6528	} while (0)
6529	#endif
6530
6531
6532	/**
6533	* Gets the IEM exception flags for the specified SVM event.
6534	*
6535	* @returns The IEM exception flags.
6536	* @param pEvent Pointer to the SVM event.
6537	*
6538	* @remarks This function currently only constructs flags required for
6539	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
6540	* and CR2 aspects of an exception are not included).
6541	*/
6542	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6543	{
6544	uint8_t const uEventType = pEvent->n.u3Type;
6545	uint32_t fIemXcptFlags;
6546	switch (uEventType)
6547	{
6548	case SVM_EVENT_EXCEPTION:
6549	/*
6550	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6551	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6552	*/
6553	if (pEvent->n.u8Vector == X86_XCPT_BP)
6554	{
6555	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
6556	break;
6557	}
6558	if (pEvent->n.u8Vector == X86_XCPT_OF)
6559	{
6560	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
6561	break;
6562	}
6563	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
6564	RT_FALL_THRU();
6565	case SVM_EVENT_NMI:
6566	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6567	break;
6568
6569	case SVM_EVENT_EXTERNAL_IRQ:
6570	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6571	break;
6572
6573	case SVM_EVENT_SOFTWARE_INT:
6574	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6575	break;
6576
6577	default:
6578	fIemXcptFlags = 0;
6579	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6580	break;
6581	}
6582	return fIemXcptFlags;
6583	}
6584
6585
6586	/**
6587	* Handle a condition that occurred while delivering an event through the guest
6588	* IDT.
6589	*
6590	* @returns VBox status code (informational error codes included).
6591	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6592	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6593	* continue execution of the guest which will delivery the \#DF.
6594	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6595	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6596	*
6597	* @param pVCpu The cross context virtual CPU structure.
6598	* @param pSvmTransient Pointer to the SVM transient structure.
6599	*
6600	* @remarks No-long-jump zone!!!
6601	*/
6602	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6603	{
6604	/** @todo r=bird: Looks like this is called on many exits and we start by
6605	* loading CR2 on the offchance that we actually have work to do here.
6606	*
6607	* HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY can surely check
6608	* pVmcb->ctrl.ExitIntInfo.n.u1Valid, can't it?
6609	*
6610	* Also, what's the deal with hmR0SvmGetCurrentVmcb() vs pSvmTransient->pVmcb?
6611	*/
6612	int rc = VINF_SUCCESS;
6613	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6614	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6615
6616	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6617	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6618	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6619
6620	/*
6621	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6622	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6623	*
6624	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6625	*/
6626	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6627	{
6628	IEMXCPTRAISE enmRaise;
6629	IEMXCPTRAISEINFO fRaiseInfo;
6630	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6631	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6632	if (fExitIsHwXcpt)
6633	{
6634	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6635	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6636	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6637	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6638	}
6639	else
6640	{
6641	/*
6642	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6643	* then we end up here.
6644	*
6645	* If the event was:
6646	* - a software interrupt, we can re-execute the instruction which will
6647	* regenerate the event.
6648	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
6649	* - a hardware exception or external interrupt, we re-inject it.
6650	*/
6651	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6652	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6653	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6654	else
6655	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6656	}
6657
6658	switch (enmRaise)
6659	{
6660	case IEMXCPTRAISE_CURRENT_XCPT:
6661	case IEMXCPTRAISE_PREV_EVENT:
6662	{
6663	/* For software interrupts, we shall re-execute the instruction. */
6664	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6665	{
6666	RTGCUINTPTR GCPtrFaultAddress = 0;
6667
6668	/* If we are re-injecting an NMI, clear NMI blocking. */
6669	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6670	CPUMClearInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
6671
6672	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6673	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6674	{
6675	pSvmTransient->fVectoringPF = true;
6676	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
6677	pVCpu->cpum.GstCtx.cr2));
6678	}
6679	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6680	&& uIdtVector == X86_XCPT_PF)
6681	{
6682	/*
6683	* If the previous exception was a #PF, we need to recover the CR2 value.
6684	* This can't happen with shadow paging.
6685	*/
6686	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
6687	}
6688
6689	/*
6690	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6691	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6692	*/
6693	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6694	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
6695	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6696
6697	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6698	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6699	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6700	}
6701	break;
6702	}
6703
6704	case IEMXCPTRAISE_REEXEC_INSTR:
6705	{
6706	Assert(rc == VINF_SUCCESS);
6707	break;
6708	}
6709
6710	case IEMXCPTRAISE_DOUBLE_FAULT:
6711	{
6712	/*
6713	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
6714	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6715	* converted into a #DF.
6716	*/
6717	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6718	{
6719	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6720	pSvmTransient->fVectoringDoublePF = true;
6721	Assert(rc == VINF_SUCCESS);
6722	}
6723	else
6724	{
6725	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
6726	hmR0SvmSetPendingXcptDF(pVCpu);
6727	rc = VINF_HM_DOUBLE_FAULT;
6728	}
6729	break;
6730	}
6731
6732	case IEMXCPTRAISE_TRIPLE_FAULT:
6733	{
6734	rc = VINF_EM_RESET;
6735	break;
6736	}
6737
6738	case IEMXCPTRAISE_CPU_HANG:
6739	{
6740	rc = VERR_EM_GUEST_CPU_HANG;
6741	break;
6742	}
6743
6744	default:
6745	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6746	}
6747	}
6748	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6749	return rc;
6750	}
6751
6752
6753	/**
6754	* Advances the guest RIP by the number of bytes specified in @a cb.
6755	*
6756	* @param pVCpu The cross context virtual CPU structure.
6757	* @param cb RIP increment value in bytes.
6758	*/
6759	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
6760	{
6761	pVCpu->cpum.GstCtx.rip += cb;
6762	CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
6763	/** @todo clear RF. */
6764	}
6765
6766
6767	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6768	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6769	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6770
6771	/** @name \#VMEXIT handlers.
6772	* @{
6773	*/
6774
6775	/**
6776	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6777	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6778	*/
6779	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6780	{
6781	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6782
6783	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6784	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6785	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6786	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6787
6788	/*
6789	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6790	* signal -before- the timer fires if the current interrupt is our own timer or a some
6791	* other host interrupt. We also cannot examine what interrupt it is until the host
6792	* actually take the interrupt.
6793	*
6794	* Going back to executing guest code here unconditionally causes random scheduling
6795	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6796	*/
6797	return VINF_EM_RAW_INTERRUPT;
6798	}
6799
6800
6801	/**
6802	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6803	*/
6804	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6805	{
6806	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6807
6808	VBOXSTRICTRC rcStrict;
6809	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6810	if (fSupportsNextRipSave)
6811	{
6812	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6813	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6814	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6815	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6816	}
6817	else
6818	{
6819	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6820	rcStrict = IEMExecOne(pVCpu);
6821	}
6822
6823	if (rcStrict == VINF_IEM_RAISED_XCPT)
6824	{
6825	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6826	rcStrict = VINF_SUCCESS;
6827	}
6828	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6829	return rcStrict;
6830	}
6831
6832
6833	/**
6834	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6835	*/
6836	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6837	{
6838	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6839
6840	VBOXSTRICTRC rcStrict;
6841	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6842	if (fSupportsNextRipSave)
6843	{
6844	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6845	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6846	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6847	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6848	}
6849	else
6850	{
6851	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6852	rcStrict = IEMExecOne(pVCpu);
6853	}
6854
6855	if (rcStrict == VINF_IEM_RAISED_XCPT)
6856	{
6857	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6858	rcStrict = VINF_SUCCESS;
6859	}
6860	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6861	return rcStrict;
6862	}
6863
6864
6865	/**
6866	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6867	*/
6868	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6869	{
6870	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6871
6872	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
6873	VBOXSTRICTRC rcStrict;
6874	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6875	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6876	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6877	if (!pExitRec)
6878	{
6879	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6880	if (fSupportsNextRipSave)
6881	{
6882	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6883	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6884	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6885	}
6886	else
6887	{
6888	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6889	rcStrict = IEMExecOne(pVCpu);
6890	}
6891
6892	if (rcStrict == VINF_IEM_RAISED_XCPT)
6893	{
6894	CPUM_ASSERT_NOT_EXTRN(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
6895	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6896	rcStrict = VINF_SUCCESS;
6897	}
6898	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6899	}
6900	else
6901	{
6902	/*
6903	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6904	*/
6905	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6906
6907	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6908	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6909
6910	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6911
6912	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6913	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6914	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6915	}
6916	return rcStrict;
6917	}
6918
6919
6920	/**
6921	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6922	*/
6923	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6924	{
6925	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6926
6927	VBOXSTRICTRC rcStrict;
6928	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6929	if (fSupportsNextRipSave)
6930	{
6931	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6932	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6933	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6934	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6935	}
6936	else
6937	{
6938	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6939	rcStrict = IEMExecOne(pVCpu);
6940	}
6941
6942	if (rcStrict == VINF_SUCCESS)
6943	pSvmTransient->fUpdateTscOffsetting = true;
6944	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6945	{
6946	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6947	rcStrict = VINF_SUCCESS;
6948	}
6949	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6950	return rcStrict;
6951	}
6952
6953
6954	/**
6955	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6956	*/
6957	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6958	{
6959	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6960
6961	VBOXSTRICTRC rcStrict;
6962	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6963	if (fSupportsNextRipSave)
6964	{
6965	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
6966	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6967	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6968	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
6969	}
6970	else
6971	{
6972	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6973	rcStrict = IEMExecOne(pVCpu);
6974	}
6975
6976	if (rcStrict == VINF_SUCCESS)
6977	pSvmTransient->fUpdateTscOffsetting = true;
6978	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6979	{
6980	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6981	rcStrict = VINF_SUCCESS;
6982	}
6983	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6984	return rcStrict;
6985	}
6986
6987
6988	/**
6989	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6990	*/
6991	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6992	{
6993	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6994
6995	VBOXSTRICTRC rcStrict;
6996	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6997	if (fSupportsNextRipSave)
6998	{
6999	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
7000	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7001	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7002	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
7003	}
7004	else
7005	{
7006	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7007	rcStrict = IEMExecOne(pVCpu);
7008	}
7009
7010	if (rcStrict == VINF_IEM_RAISED_XCPT)
7011	{
7012	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7013	rcStrict = VINF_SUCCESS;
7014	}
7015	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7016	return rcStrict;
7017	}
7018
7019
7020	/**
7021	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
7022	*/
7023	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7024	{
7025	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7026	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7027
7028	VBOXSTRICTRC rcStrict;
7029	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7030	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7031	if ( fSupportsDecodeAssists
7032	&& fSupportsNextRipSave)
7033	{
7034	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7035	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7036	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7037	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
7038	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
7039	}
7040	else
7041	{
7042	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7043	rcStrict = IEMExecOne(pVCpu);
7044	}
7045
7046	if (rcStrict == VINF_IEM_RAISED_XCPT)
7047	{
7048	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7049	rcStrict = VINF_SUCCESS;
7050	}
7051	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7052	return VBOXSTRICTRC_VAL(rcStrict);
7053	}
7054
7055
7056	/**
7057	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
7058	*/
7059	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7060	{
7061	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7062
7063	VBOXSTRICTRC rcStrict;
7064	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7065	if (fSupportsNextRipSave)
7066	{
7067	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7068	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7069	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7070	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
7071	}
7072	else
7073	{
7074	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7075	rcStrict = IEMExecOne(pVCpu);
7076	}
7077
7078	if ( rcStrict == VINF_EM_HALT
7079	\|\| rcStrict == VINF_SUCCESS)
7080	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
7081	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7082	{
7083	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7084	rcStrict = VINF_SUCCESS;
7085	}
7086	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7087	if (rcStrict != VINF_SUCCESS)
7088	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
7089	return VBOXSTRICTRC_VAL(rcStrict);;
7090	}
7091
7092
7093	/**
7094	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
7095	*/
7096	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7097	{
7098	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7099
7100	/*
7101	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
7102	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
7103	* segment override prefix or other prefixes might be used, in which case we fallback to
7104	* IEMExecOne() to figure out.
7105	*/
7106	VBOXSTRICTRC rcStrict;
7107	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7108	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
7109	if (cbInstr)
7110	{
7111	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7112	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
7113	}
7114	else
7115	{
7116	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7117	rcStrict = IEMExecOne(pVCpu);
7118	}
7119
7120	if (rcStrict == VINF_IEM_RAISED_XCPT)
7121	{
7122	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7123	rcStrict = VINF_SUCCESS;
7124	}
7125	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7126	return rcStrict;
7127	}
7128
7129
7130	/**
7131	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
7132	*/
7133	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7134	{
7135	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7136
7137	VBOXSTRICTRC rcStrict;
7138	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7139	if (fSupportsNextRipSave)
7140	{
7141	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7142	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7143	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7144	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
7145	}
7146	else
7147	{
7148	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7149	rcStrict = IEMExecOne(pVCpu);
7150	}
7151
7152	if ( rcStrict == VINF_EM_HALT
7153	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7154	rcStrict = VINF_SUCCESS;
7155	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7156	{
7157	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7158	rcStrict = VINF_SUCCESS;
7159	}
7160	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7161	return rcStrict;
7162	}
7163
7164
7165	/**
7166	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
7167	* \#VMEXIT.
7168	*/
7169	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7170	{
7171	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7172	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7173	return VINF_EM_RESET;
7174	}
7175
7176
7177	/**
7178	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
7179	*/
7180	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7181	{
7182	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7183	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7184	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
7185	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
7186	RT_NOREF(pVmcb);
7187	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7188	return VERR_SVM_UNEXPECTED_EXIT;
7189	}
7190
7191
7192	/**
7193	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
7194	*/
7195	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7196	{
7197	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7198
7199	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7200	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7201	#ifdef VBOX_WITH_STATISTICS
7202	switch (pSvmTransient->u64ExitCode)
7203	{
7204	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
7205	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
7206	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
7207	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
7208	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
7209	}
7210	#endif
7211
7212	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7213	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7214	if ( fSupportsDecodeAssists
7215	&& fSupportsNextRipSave)
7216	{
7217	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7218	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7219	if (fMovCRx)
7220	{
7221	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
7222	\| CPUMCTX_EXTRN_APIC_TPR);
7223	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7224	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
7225	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7226	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
7227	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7228	return VBOXSTRICTRC_VAL(rcStrict);
7229	}
7230	/* else: SMSW instruction, fall back below to IEM for this. */
7231	}
7232
7233	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7234	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7235	AssertMsg( rcStrict == VINF_SUCCESS
7236	\|\| rcStrict == VINF_PGM_SYNC_CR3
7237	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
7238	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7239	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
7240	if (rcStrict == VINF_IEM_RAISED_XCPT)
7241	{
7242	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7243	rcStrict = VINF_SUCCESS;
7244	}
7245	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7246	return rcStrict;
7247	}
7248
7249
7250	/**
7251	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
7252	*/
7253	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7254	{
7255	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7256
7257	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
7258	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
7259	Assert(iCrReg <= 15);
7260
7261	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
7262	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7263	bool fDecodedInstr = false;
7264	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7265	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7266	if ( fSupportsDecodeAssists
7267	&& fSupportsNextRipSave)
7268	{
7269	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7270	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7271	if (fMovCRx)
7272	{
7273	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
7274	\| CPUMCTX_EXTRN_APIC_TPR);
7275	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7276	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7277	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
7278	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
7279	fDecodedInstr = true;
7280	}
7281	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
7282	}
7283
7284	if (!fDecodedInstr)
7285	{
7286	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7287	Log4Func(("iCrReg=%#x\n", iCrReg));
7288	rcStrict = IEMExecOne(pVCpu);
7289	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
7290	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
7291	rcStrict = VERR_EM_INTERPRETER;
7292	}
7293
7294	if (rcStrict == VINF_SUCCESS)
7295	{
7296	switch (iCrReg)
7297	{
7298	case 0:
7299	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
7300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
7301	break;
7302
7303	case 2:
7304	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
7305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
7306	break;
7307
7308	case 3:
7309	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
7310	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
7311	break;
7312
7313	case 4:
7314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
7315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
7316	break;
7317
7318	case 8:
7319	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7320	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
7321	break;
7322
7323	default:
7324	{
7325	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
7326	pSvmTransient->u64ExitCode, iCrReg));
7327	break;
7328	}
7329	}
7330	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7331	}
7332	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7333	{
7334	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7335	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7336	rcStrict = VINF_SUCCESS;
7337	}
7338	else
7339	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
7340	return rcStrict;
7341	}
7342
7343
7344	/**
7345	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
7346	*
7347	* @returns Strict VBox status code.
7348	* @param pVCpu The cross context virtual CPU structure.
7349	* @param pVmcb Pointer to the VM control block.
7350	*/
7351	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
7352	{
7353	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7354	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
7355
7356	VBOXSTRICTRC rcStrict;
7357	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7358	if (fSupportsNextRipSave)
7359	{
7360	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
7361	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
7362	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7363	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7364	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
7365	}
7366	else
7367	{
7368	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7369	rcStrict = IEMExecOne(pVCpu);
7370	}
7371
7372	AssertMsg( rcStrict == VINF_SUCCESS
7373	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7374	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
7375	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7376
7377	if (rcStrict == VINF_IEM_RAISED_XCPT)
7378	{
7379	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7380	rcStrict = VINF_SUCCESS;
7381	}
7382	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7383	return rcStrict;
7384	}
7385
7386
7387	/**
7388	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
7389	*
7390	* @returns Strict VBox status code.
7391	* @param pVCpu The cross context virtual CPU structure.
7392	* @param pVmcb Pointer to the VM control block.
7393	* @param pSvmTransient Pointer to the SVM-transient structure.
7394	*/
7395	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
7396	{
7397	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7398	uint32_t const idMsr = pCtx->ecx;
7399	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7400	Log4Func(("idMsr=%#RX32\n", idMsr));
7401
7402	/*
7403	* Handle TPR patching MSR writes.
7404	* We utilitize the LSTAR MSR for patching.
7405	*/
7406	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7407	if ( idMsr == MSR_K8_LSTAR
7408	&& pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
7409	{
7410	unsigned cbInstr;
7411	if (fSupportsNextRipSave)
7412	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7413	else
7414	{
7415	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
7416	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
7417	if ( rc == VINF_SUCCESS
7418	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
7419	Assert(cbInstr > 0);
7420	else
7421	cbInstr = 0;
7422	}
7423
7424	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
7425	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
7426	{
7427	int rc = PDMApicSetTpr(pVCpu, pCtx->eax & 0xff);
7428	AssertRCReturn(rc, rc);
7429	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7430	}
7431
7432	int rc = VINF_SUCCESS;
7433	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7434	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7435	return rc;
7436	}
7437
7438	/*
7439	* Handle regular MSR writes.
7440	*/
7441	VBOXSTRICTRC rcStrict;
7442	if (fSupportsNextRipSave)
7443	{
7444	/** @todo Optimize this: We don't need to get much of the MSR state here
7445	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
7446	* clear the applicable extern flags. */
7447	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7448	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7449	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
7450	}
7451	else
7452	{
7453	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7454	rcStrict = IEMExecOne(pVCpu);
7455	}
7456
7457	AssertMsg( rcStrict == VINF_SUCCESS
7458	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7459	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
7460	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7461
7462	if (rcStrict == VINF_SUCCESS)
7463	{
7464	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
7465	if ( idMsr >= MSR_IA32_X2APIC_START
7466	&& idMsr <= MSR_IA32_X2APIC_END)
7467	{
7468	/*
7469	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
7470	* When full APIC register virtualization is implemented we'll have to make sure
7471	* APIC state is saved from the VMCB before IEM changes it.
7472	*/
7473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7474	}
7475	else
7476	{
7477	switch (idMsr)
7478	{
7479	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
7480	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
7481	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
7482	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
7483	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7484	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7485	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7486	}
7487	}
7488	}
7489	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7490	{
7491	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7492	rcStrict = VINF_SUCCESS;
7493	}
7494	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7495	return rcStrict;
7496	}
7497
7498
7499	/**
7500	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
7501	* \#VMEXIT.
7502	*/
7503	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7504	{
7505	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7506
7507	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7508	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
7509	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
7510
7511	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
7512	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
7513	}
7514
7515
7516	/**
7517	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
7518	*/
7519	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7520	{
7521	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7522	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7523
7524	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7525
7526	/** @todo Stepping with nested-guest. */
7527	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7528	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7529	{
7530	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
7531	if (pSvmTransient->fWasGuestDebugStateActive)
7532	{
7533	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
7534	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7535	return VERR_SVM_UNEXPECTED_EXIT;
7536	}
7537
7538	/*
7539	* Lazy DR0-3 loading.
7540	*/
7541	if (!pSvmTransient->fWasHyperDebugStateActive)
7542	{
7543	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
7544	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
7545
7546	/* Don't intercept DRx read and writes. */
7547	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7548	pVmcb->ctrl.u16InterceptRdDRx = 0;
7549	pVmcb->ctrl.u16InterceptWrDRx = 0;
7550	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
7551
7552	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7553	VMMRZCallRing3Disable(pVCpu);
7554	HM_DISABLE_PREEMPT(pVCpu);
7555
7556	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7557	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
7558	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7559
7560	HM_RESTORE_PREEMPT();
7561	VMMRZCallRing3Enable(pVCpu);
7562
7563	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7564	return VINF_SUCCESS;
7565	}
7566	}
7567
7568	/*
7569	* Interpret the read/writing of DRx.
7570	*/
7571	/** @todo Decode assist. */
7572	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
7573	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
7574	if (RT_LIKELY(rc == VINF_SUCCESS))
7575	{
7576	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
7577	/** @todo CPUM should set this flag! */
7578	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
7579	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7580	}
7581	else
7582	Assert(rc == VERR_EM_INTERPRETER);
7583	return rc;
7584	}
7585
7586
7587	/**
7588	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
7589	*/
7590	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7591	{
7592	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7593	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
7594	VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
7595	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7596	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
7597	return rc;
7598	}
7599
7600
7601	/**
7602	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
7603	*/
7604	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7605	{
7606	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7607	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7608
7609	/** @todo decode assists... */
7610	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7611	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7612	{
7613	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7614	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7615	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
7616	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7617	{
7618	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7619	hmR0SvmUpdateVmRunFunction(pVCpu);
7620	}
7621	}
7622	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7623	{
7624	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7625	rcStrict = VINF_SUCCESS;
7626	}
7627	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7628	return rcStrict;
7629	}
7630
7631
7632	/**
7633	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
7634	*/
7635	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7636	{
7637	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7638	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
7639
7640	/* I/O operation lookup arrays. */
7641	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7642	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7643	the result (in AL/AX/EAX). */
7644	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7645	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7646	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7647
7648	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7649
7650	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7651	SVMIOIOEXITINFO IoExitInfo;
7652	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7653	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7654	uint32_t cbValue = s_aIOSize[uIOWidth];
7655	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7656
7657	if (RT_UNLIKELY(!cbValue))
7658	{
7659	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7660	return VERR_EM_INTERPRETER;
7661	}
7662
7663	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
7664	VBOXSTRICTRC rcStrict;
7665	PCEMEXITREC pExitRec = NULL;
7666	if ( !pVCpu->hm.s.fSingleInstruction
7667	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7668	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7669	!IoExitInfo.n.u1Str
7670	? IoExitInfo.n.u1Type == SVM_IOIO_READ
7671	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7672	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7673	: IoExitInfo.n.u1Type == SVM_IOIO_READ
7674	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7675	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7676	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7677	if (!pExitRec)
7678	{
7679	bool fUpdateRipAlready = false;
7680	if (IoExitInfo.n.u1Str)
7681	{
7682	/* INS/OUTS - I/O String instruction. */
7683	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
7684	* in EXITINFO1? Investigate once this thing is up and running. */
7685	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7686	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7687	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7688	static IEMMODE const s_aenmAddrMode[8] =
7689	{
7690	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7691	};
7692	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7693	if (enmAddrMode != (IEMMODE)-1)
7694	{
7695	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7696	if (cbInstr <= 15 && cbInstr >= 1)
7697	{
7698	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7699	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7700	{
7701	/* Don't know exactly how to detect whether u3Seg is valid, currently
7702	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7703	2384 Opterons when only checking NRIP. */
7704	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7705	if ( fSupportsNextRipSave
7706	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7707	{
7708	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
7709	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7710	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7711	IoExitInfo.n.u3Seg, true /fIoChecked/);
7712	}
7713	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7714	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7715	X86_SREG_DS, true /fIoChecked/);
7716	else
7717	rcStrict = IEMExecOne(pVCpu);
7718	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7719	}
7720	else
7721	{
7722	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7723	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7724	true /fIoChecked/);
7725	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7726	}
7727	}
7728	else
7729	{
7730	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7731	rcStrict = IEMExecOne(pVCpu);
7732	}
7733	}
7734	else
7735	{
7736	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7737	rcStrict = IEMExecOne(pVCpu);
7738	}
7739	fUpdateRipAlready = true;
7740	if (rcStrict == VINF_IEM_RAISED_XCPT)
7741	{
7742	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7743	rcStrict = VINF_SUCCESS;
7744	}
7745	}
7746	else
7747	{
7748	/* IN/OUT - I/O instruction. */
7749	Assert(!IoExitInfo.n.u1Rep);
7750
7751	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7752	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7753	{
7754	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7755	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7756	&& !pCtx->eflags.Bits.u1TF)
7757	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7758	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7759	}
7760	else
7761	{
7762	rcStrict = VERR_GCM_NOT_HANDLED;
7763	if (GCMIsInterceptingIOPortRead(pVCpu, IoExitInfo.n.u16Port, cbValue))
7764	{
7765	rcStrict = GCMInterceptedIOPortRead(pVCpu, pCtx, IoExitInfo.n.u16Port, cbValue);
7766	if (rcStrict == VINF_GCM_HANDLED_ADVANCE_RIP \|\| rcStrict == VINF_GCM_HANDLED)
7767	{
7768	fUpdateRipAlready = rcStrict != VINF_GCM_HANDLED_ADVANCE_RIP;
7769	rcStrict = VINF_SUCCESS;
7770	}
7771	else
7772	Assert(rcStrict == VERR_GCM_NOT_HANDLED);
7773	}
7774
7775	if (RT_LIKELY(rcStrict == VERR_GCM_NOT_HANDLED))
7776	{
7777	uint32_t u32Val = 0;
7778	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7779	if (IOM_SUCCESS(rcStrict))
7780	{
7781	/* Save result of I/O IN instr. in AL/AX/EAX. */
7782	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7783	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7784	}
7785	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7786	&& !pCtx->eflags.Bits.u1TF)
7787	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7788	}
7789
7790	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7791	}
7792	}
7793
7794	if (IOM_SUCCESS(rcStrict))
7795	{
7796	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7797	if (!fUpdateRipAlready)
7798	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7799
7800	/*
7801	* If any I/O breakpoints are armed, we need to check if one triggered
7802	* and take appropriate action.
7803	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7804	*/
7805	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7806	* execution engines about whether hyper BPs and such are pending. */
7807	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7808	uint32_t const uDr7 = pCtx->dr[7];
7809	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7810	&& X86_DR7_ANY_RW_IO(uDr7)
7811	&& (pCtx->cr4 & X86_CR4_DE))
7812	\|\| DBGFBpIsHwIoArmed(pVM)))
7813	{
7814	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7815	VMMRZCallRing3Disable(pVCpu);
7816	HM_DISABLE_PREEMPT(pVCpu);
7817
7818	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7819	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7820
7821	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7822	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7823	{
7824	/* Raise #DB. */
7825	pVmcb->guest.u64DR6 = pCtx->dr[6];
7826	pVmcb->guest.u64DR7 = pCtx->dr[7];
7827	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7828	hmR0SvmSetPendingXcptDB(pVCpu);
7829	}
7830	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7831	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7832	else if ( rcStrict2 != VINF_SUCCESS
7833	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7834	rcStrict = rcStrict2;
7835	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7836
7837	HM_RESTORE_PREEMPT();
7838	VMMRZCallRing3Enable(pVCpu);
7839	}
7840
7841	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7842	}
7843	#ifdef VBOX_STRICT
7844	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7845	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7846	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7847	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7848	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7849	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7850	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7851	else
7852	{
7853	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7854	* statuses, that the VMM device and some others may return. See
7855	* IOM_SUCCESS() for guidance. */
7856	AssertMsg( RT_FAILURE(rcStrict)
7857	\|\| rcStrict == VINF_SUCCESS
7858	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7859	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7860	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7861	\|\| rcStrict == VINF_EM_DBG_STEPPED
7862	\|\| rcStrict == VINF_EM_RAW_TO_R3
7863	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7864	}
7865	#endif
7866	}
7867	else
7868	{
7869	/*
7870	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7871	*/
7872	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7873	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7874	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7875	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7876	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7877	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7878	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7879
7880	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7881	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7882
7883	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7884	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7885	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7886	}
7887	return rcStrict;
7888	}
7889
7890
7891	/**
7892	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7893	*/
7894	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7895	{
7896	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7897	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7898	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7899
7900	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7901	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7902	Assert(pVM->hmr0.s.fNestedPaging);
7903
7904	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7905	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7906	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7907	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7908	thus intentionally not copied into u32ErrCode. */
7909
7910	Log4Func(("#NPF at CS:RIP=%04x:%RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7911	u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
7912
7913	/*
7914	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7915	*/
7916	if ( pVM->hm.s.fTprPatchingAllowed
7917	&& (GCPhysFaultAddr & GUEST_PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7918	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7919	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7920	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7921	&& !CPUMIsGuestInLongModeEx(pCtx)
7922	&& !CPUMGetGuestCPL(pVCpu)
7923	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7924	{
7925	RTGCPHYS GCPhysApicBase = PDMApicGetBaseMsrNoCheck(pVCpu);
7926	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
7927
7928	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7929	{
7930	/* Only attempt to patch the instruction once. */
7931	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7932	if (!pPatch)
7933	return VINF_EM_HM_PATCH_TPR_INSTR;
7934	}
7935	}
7936
7937	/*
7938	* Determine the nested paging mode.
7939	*/
7940	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7941	PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
7942
7943	/*
7944	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7945	*/
7946	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7947	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7948	{
7949	/*
7950	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7951	* injecting the original pending event would most likely cause the same MMIO #NPF.
7952	*/
7953	if (pVCpu->hm.s.Event.fPending)
7954	{
7955	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7956	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7957	}
7958
7959	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
7960	VBOXSTRICTRC rcStrict;
7961	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7962	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7963	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7964	if (!pExitRec)
7965	{
7966
7967	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, pCtx, GCPhysFaultAddr, u32ErrCode);
7968
7969	/*
7970	* If we succeed, resume guest execution.
7971	*
7972	* If we fail in interpreting the instruction because we couldn't get the guest
7973	* physical address of the page containing the instruction via the guest's page
7974	* tables (we would invalidate the guest page in the host TLB), resume execution
7975	* which would cause a guest page fault to let the guest handle this weird case.
7976	*
7977	* See @bugref{6043}.
7978	*/
7979	if ( rcStrict == VINF_SUCCESS
7980	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7981	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
7982	{
7983	/* Successfully handled MMIO operation. */
7984	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7985	rcStrict = VINF_SUCCESS;
7986	}
7987	}
7988	else
7989	{
7990	/*
7991	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7992	*/
7993	Assert(pCtx == &pVCpu->cpum.GstCtx);
7994	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7995	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
7996	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
7997
7998	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7999	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8000
8001	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8002	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8003	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8004	}
8005	return rcStrict;
8006	}
8007
8008	/*
8009	* Nested page-fault.
8010	*/
8011	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
8012	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, pCtx, GCPhysFaultAddr);
8013	TRPMResetTrap(pVCpu);
8014
8015	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
8016
8017	/*
8018	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
8019	*/
8020	if ( rc == VINF_SUCCESS
8021	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8022	\|\| rc == VERR_PAGE_NOT_PRESENT)
8023	{
8024	/* We've successfully synced our shadow page tables. */
8025	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8026	rc = VINF_SUCCESS;
8027	}
8028
8029	/*
8030	* If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
8031	* re-inject the original event.
8032	*/
8033	if (pVCpu->hm.s.Event.fPending)
8034	{
8035	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
8036
8037	/*
8038	* If the #NPF handler requested emulation of the instruction, ignore it.
8039	* We need to re-inject the original event so as to not lose it.
8040	* Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
8041	* LiveCD is broken for other reasons).
8042	*/
8043	if (rc == VINF_EM_RAW_EMULATE_INSTR)
8044	rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
8045	}
8046
8047	return rc;
8048	}
8049
8050
8051	/**
8052	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
8053	* \#VMEXIT.
8054	*/
8055	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8056	{
8057	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8058	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8059
8060	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
8061	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8062	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
8063
8064	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
8065	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8066	return VINF_SUCCESS;
8067	}
8068
8069
8070	/**
8071	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
8072	* \#VMEXIT.
8073	*/
8074	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8075	{
8076	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8077	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8078
8079	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
8080	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8081	#endif
8082
8083	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
8084	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
8085	{
8086	/*
8087	* AMD-V provides us with the exception which caused the TS; we collect
8088	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
8089	*/
8090	Log4Func(("TS occurred during event delivery\n"));
8091	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8092	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8093	}
8094
8095	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8096	* emulation. */
8097	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8098	return VERR_EM_INTERPRETER;
8099	}
8100
8101
8102	/**
8103	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8104	*/
8105	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8106	{
8107	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8108	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8109
8110	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8111	if (pVM->hm.s.fTprPatchingAllowed)
8112	{
8113	int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
8114	if (rc != VERR_NOT_FOUND)
8115	{
8116	Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
8117	return rc;
8118	}
8119	}
8120
8121	if (EMAreHypercallInstructionsEnabled(pVCpu))
8122	{
8123	unsigned cbInstr;
8124	if (hmR0SvmSupportsNextRipSave(pVCpu))
8125	{
8126	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8127	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8128	}
8129	else
8130	{
8131	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8132	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8133	if ( rc == VINF_SUCCESS
8134	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
8135	Assert(cbInstr > 0);
8136	else
8137	cbInstr = 0;
8138	}
8139
8140	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8141	if (RT_SUCCESS(rcStrict))
8142	{
8143	/* Only update the RIP if we're continuing guest execution and not in the case
8144	of say VINF_GIM_R3_HYPERCALL. */
8145	if (rcStrict == VINF_SUCCESS)
8146	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8147
8148	return VBOXSTRICTRC_VAL(rcStrict);
8149	}
8150	else
8151	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
8152	}
8153
8154	hmR0SvmSetPendingXcptUD(pVCpu);
8155	return VINF_SUCCESS;
8156	}
8157
8158
8159	/**
8160	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8161	*/
8162	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8163	{
8164	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8165
8166	unsigned cbInstr;
8167	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8168	if (fSupportsNextRipSave)
8169	{
8170	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8171	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8172	}
8173	else
8174	{
8175	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8176	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8177	if ( rc == VINF_SUCCESS
8178	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
8179	Assert(cbInstr > 0);
8180	else
8181	cbInstr = 0;
8182	}
8183
8184	/** @todo The guest has likely hit a contended spinlock. We might want to
8185	* poke a schedule different guest VCPU. */
8186	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8187	return VINF_EM_RAW_INTERRUPT;
8188	}
8189
8190
8191	/**
8192	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
8193	* \#VMEXIT.
8194	*/
8195	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8196	{
8197	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8198	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
8199	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
8200
8201	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
8202	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8203	}
8204
8205
8206	/**
8207	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
8208	*/
8209	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8210	{
8211	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8212
8213	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
8214	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8215	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
8216
8217	/* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
8218	* deliver a pending NMI right after. If the IRET faults, an NMI can come before the
8219	* handler executes. Yes, x86 is ugly.
8220	*/
8221	return VINF_EM_RAW_EMULATE_INSTR;
8222	}
8223
8224
8225	/**
8226	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
8227	* Conditional \#VMEXIT.
8228	*/
8229	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8230	{
8231	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8232	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8233	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8234
8235	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
8236	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8237	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8238	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8239	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
8240	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
8241
8242	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
8243	if (pVM->hmr0.s.fNestedPaging)
8244	{
8245	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8246	if ( !pSvmTransient->fVectoringDoublePF
8247	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8248	{
8249	/* A genuine guest #PF, reflect it to the guest. */
8250	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8251	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
8252	uFaultAddress, uErrCode));
8253	}
8254	else
8255	{
8256	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8257	hmR0SvmSetPendingXcptDF(pVCpu);
8258	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
8259	}
8260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8261	return VINF_SUCCESS;
8262	}
8263	#endif
8264
8265	Assert(!pVM->hmr0.s.fNestedPaging);
8266
8267	/*
8268	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
8269	*/
8270	if ( pVM->hm.s.fTprPatchingAllowed
8271	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
8272	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
8273	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8274	&& !CPUMIsGuestInLongModeEx(pCtx)
8275	&& !CPUMGetGuestCPL(pVCpu)
8276	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8277	{
8278	RTGCPHYS GCPhysApicBase;
8279	GCPhysApicBase = PDMApicGetBaseMsrNoCheck(pVCpu);
8280	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
8281
8282	/* Check if the page at the fault-address is the APIC base. */
8283	PGMPTWALK Walk;
8284	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, &Walk);
8285	if ( rc2 == VINF_SUCCESS
8286	&& Walk.GCPhys == GCPhysApicBase)
8287	{
8288	/* Only attempt to patch the instruction once. */
8289	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
8290	if (!pPatch)
8291	return VINF_EM_HM_PATCH_TPR_INSTR;
8292	}
8293	}
8294
8295	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
8296	pCtx->rip, uErrCode, pCtx->cr3));
8297
8298	/*
8299	* If it's a vectoring #PF, emulate injecting the original event injection as
8300	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
8301	* event injection that caused a #PF. See @bugref{6607}.
8302	*/
8303	if (pSvmTransient->fVectoringPF)
8304	{
8305	Assert(pVCpu->hm.s.Event.fPending);
8306	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8307	}
8308
8309	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
8310	int rc = PGMTrap0eHandler(pVCpu, uErrCode, pCtx, (RTGCPTR)uFaultAddress);
8311
8312	Log4Func(("#PF: rc=%Rrc\n", rc));
8313
8314	if (rc == VINF_SUCCESS)
8315	{
8316	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8317	TRPMResetTrap(pVCpu);
8318	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8319	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8320	return rc;
8321	}
8322
8323	if (rc == VINF_EM_RAW_GUEST_TRAP)
8324	{
8325	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8326
8327	/*
8328	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
8329	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
8330	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
8331	*/
8332	if ( !pSvmTransient->fVectoringDoublePF
8333	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8334	{
8335	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
8336	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
8337	TRPMResetTrap(pVCpu);
8338
8339	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8340	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
8341	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8342	&& CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
8343	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
8344	#endif
8345
8346	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8347	}
8348	else
8349	{
8350	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8351	TRPMResetTrap(pVCpu);
8352	hmR0SvmSetPendingXcptDF(pVCpu);
8353	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
8354	}
8355
8356	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8357	return VINF_SUCCESS;
8358	}
8359
8360	TRPMResetTrap(pVCpu);
8361	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8362	return rc;
8363	}
8364
8365
8366
8367	/**
8368	* \#VMEXIT handler for division overflow exceptions (SVM_EXIT_XCPT_1).
8369	* Conditional \#VMEXIT.
8370	*/
8371	HMSVM_EXIT_DECL hmR0SvmExitXcptDE(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8372	{
8373	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8374	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8376
8377	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8378	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8379	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8380
8381	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8382	if (pVCpu->hm.s.fGCMTrapXcptDE)
8383	{
8384	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8385	rc = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx);
8386	AssertMsg(rc == VINF_SUCCESS /* restart / \|\| rc == VERR_NOT_FOUND / deliver exception */, ("rc=%Rrc\n", rc));
8387	}
8388
8389	/* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
8390	if (RT_FAILURE(rc))
8391	hmR0SvmSetPendingXcptDE(pVCpu);
8392
8393	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8394	return VINF_SUCCESS;
8395	}
8396
8397
8398	/**
8399	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
8400	* Conditional \#VMEXIT.
8401	*/
8402	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8403	{
8404	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8405	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8406	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8407
8408	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8409	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8410	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8411
8412	/** @todo if we accumulate more optional stuff here, we ought to combine the
8413	* reading of opcode bytes to avoid doing more than once. */
8414
8415	VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8416	if (pVCpu->hm.s.fGIMTrapXcptUD)
8417	{
8418	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8419	uint8_t cbInstr = 0;
8420	rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8421	if (rcStrict == VINF_SUCCESS)
8422	{
8423	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
8424	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8425	rcStrict = VINF_SUCCESS;
8426	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8427	}
8428	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
8429	rcStrict = VINF_SUCCESS;
8430	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
8431	rcStrict = VINF_GIM_R3_HYPERCALL;
8432	else
8433	{
8434	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8435	rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8436	}
8437	}
8438
8439	if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
8440	{
8441	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
8442	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8443	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8444	{
8445	/* Ideally, IEM should just handle all these special #UD situations, but
8446	we don't quite trust things to behave optimially when doing that. So,
8447	for now we'll restrict ourselves to a handful of possible sysenter and
8448	sysexit encodings that we filter right here. */
8449	uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
8450	uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
8451	uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
8452	uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
8453	RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
8454	if (cbInstr < cbMin \|\| cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
8455	{
8456	cbInstr = cbMin;
8457	int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
8458	AssertRCStmt(rc2, cbInstr = 0);
8459	}
8460	else
8461	memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
8462	if ( cbInstr == 0 /* read error */
8463	\|\| (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
8464	\|\| ( uCpl == 0
8465	&& ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
8466	\|\| ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
8467	&& (abInstr[0] & (X86_OP_REX_W \| 0xf0)) == X86_OP_REX_W))))
8468	{
8469	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8470	\| CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
8471	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8472	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, GCPtrInstr, abInstr, cbInstr);
8473	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
8474	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
8475	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
8476	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8477	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
8478	if (rcStrict == VINF_IEM_RAISED_XCPT)
8479	rcStrict = VINF_SUCCESS;
8480	return rcStrict;
8481	}
8482	Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8483	}
8484	else
8485	Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8486	}
8487
8488	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
8489	if (RT_FAILURE(rcStrict))
8490	{
8491	hmR0SvmSetPendingXcptUD(pVCpu);
8492	rcStrict = VINF_SUCCESS;
8493	}
8494
8495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8496	return rcStrict;
8497	}
8498
8499
8500	/**
8501	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
8502	* Conditional \#VMEXIT.
8503	*/
8504	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8505	{
8506	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8507	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8508	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8509
8510	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8511	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8512
8513	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8514	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8515
8516	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8517
8518	if (!(pCtx->cr0 & X86_CR0_NE))
8519	{
8520	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8521	unsigned cbInstr;
8522	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8523	if (RT_SUCCESS(rc))
8524	{
8525	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
8526	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8527	if (RT_SUCCESS(rc))
8528	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8529	}
8530	else
8531	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
8532	return rc;
8533	}
8534
8535	hmR0SvmSetPendingXcptMF(pVCpu);
8536	return VINF_SUCCESS;
8537	}
8538
8539
8540	/**
8541	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
8542	* \#VMEXIT.
8543	*/
8544	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8545	{
8546	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8547	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8548	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8549	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8550
8551	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
8552	{
8553	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
8554	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8555	}
8556
8557	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8558
8559	/*
8560	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
8561	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
8562	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
8563	*/
8564	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8565	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8566	int rc = DBGFTrap01Handler(pVM, pVCpu, &pVCpu->cpum.GstCtx, pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
8567	if (rc == VINF_EM_RAW_GUEST_TRAP)
8568	{
8569	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
8570	if (CPUMIsHyperDebugStateActive(pVCpu))
8571	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
8572
8573	/* Reflect the exception back to the guest. */
8574	hmR0SvmSetPendingXcptDB(pVCpu);
8575	rc = VINF_SUCCESS;
8576	}
8577
8578	/*
8579	* Update DR6.
8580	*/
8581	if (CPUMIsHyperDebugStateActive(pVCpu))
8582	{
8583	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
8584	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
8585	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
8586	}
8587	else
8588	{
8589	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
8590	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
8591	}
8592
8593	return rc;
8594	}
8595
8596
8597	/**
8598	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
8599	* Conditional \#VMEXIT.
8600	*/
8601	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8602	{
8603	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8604	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8605	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
8606
8607	SVMEVENT Event;
8608	Event.u = 0;
8609	Event.n.u1Valid = 1;
8610	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8611	Event.n.u8Vector = X86_XCPT_AC;
8612	Event.n.u1ErrorCodeValid = 1;
8613	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8614	return VINF_SUCCESS;
8615	}
8616
8617
8618	/**
8619	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8620	* Conditional \#VMEXIT.
8621	*/
8622	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8623	{
8624	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8625	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8626	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8627	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8628
8629	VBOXSTRICTRC rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx);
8630	if (rc == VINF_EM_RAW_GUEST_TRAP)
8631	{
8632	SVMEVENT Event;
8633	Event.u = 0;
8634	Event.n.u1Valid = 1;
8635	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8636	Event.n.u8Vector = X86_XCPT_BP;
8637	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8638	rc = VINF_SUCCESS;
8639	}
8640
8641	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
8642	return rc;
8643	}
8644
8645
8646	/**
8647	* Hacks its way around the lovely mesa driver's backdoor accesses.
8648	*
8649	* @sa hmR0VmxHandleMesaDrvGp
8650	*/
8651	static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8652	{
8653	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK);
8654	Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
8655	pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
8656	RT_NOREF(pCtx, pVmcb);
8657
8658	/* For now we'll just skip the instruction. */
8659	hmR0SvmAdvanceRip(pVCpu, 1);
8660	return VINF_SUCCESS;
8661	}
8662
8663
8664	/**
8665	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
8666	* backdoor logging w/o checking what it is running inside.
8667	*
8668	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
8669	* backdoor port and magic numbers loaded in registers.
8670	*
8671	* @returns true if it is, false if it isn't.
8672	* @sa hmR0VmxIsMesaDrvGp
8673	*/
8674	DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8675	{
8676	/* Check magic and port. */
8677	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
8678	/Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));/
8679	if (pCtx->dx != UINT32_C(0x5658))
8680	return false;
8681	if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
8682	return false;
8683
8684	/* Check that it is #GP(0). */
8685	if (pVmcb->ctrl.u64ExitInfo1 != 0)
8686	return false;
8687
8688	/* Flat ring-3 CS. */
8689	/Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));/
8690	if (pVmcb->guest.u8CPL != 3)
8691	return false;
8692	if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
8693	return false;
8694
8695	/* 0xed: IN eAX,dx */
8696	if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
8697	{
8698	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_GPRS_MASK
8699	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8700	uint8_t abInstr[1];
8701	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
8702	/Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); /
8703	if (RT_FAILURE(rc))
8704	return false;
8705	if (abInstr[0] != 0xed)
8706	return false;
8707	}
8708	else
8709	{
8710	/Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));/
8711	if (pVmcb->ctrl.abInstr[0] != 0xed)
8712	return false;
8713	}
8714	return true;
8715	}
8716
8717
8718	/**
8719	* \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
8720	* Conditional \#VMEXIT.
8721	*/
8722	HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8723	{
8724	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8725	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8726	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8727
8728	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8729	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8730
8731	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8732	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
8733	\|\| !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
8734	{
8735	SVMEVENT Event;
8736	Event.u = 0;
8737	Event.n.u1Valid = 1;
8738	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8739	Event.n.u8Vector = X86_XCPT_GP;
8740	Event.n.u1ErrorCodeValid = 1;
8741	Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
8742	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8743	return VINF_SUCCESS;
8744	}
8745	return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
8746	}
8747
8748
8749	/**
8750	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
8751	*/
8752	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8753	{
8754	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8755	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8756
8757	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8758	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
8759	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
8760	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8761	Assert(uVector <= X86_XCPT_LAST);
8762	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
8763
8764	SVMEVENT Event;
8765	Event.u = 0;
8766	Event.n.u1Valid = 1;
8767	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8768	Event.n.u8Vector = uVector;
8769	switch (uVector)
8770	{
8771	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
8772	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
8773	case X86_XCPT_DF:
8774	case X86_XCPT_TS:
8775	case X86_XCPT_NP:
8776	case X86_XCPT_SS:
8777	case X86_XCPT_GP:
8778	case X86_XCPT_AC:
8779	{
8780	Event.n.u1ErrorCodeValid = 1;
8781	Event.n.u32ErrorCode = uErrCode;
8782	break;
8783	}
8784	}
8785
8786	#ifdef VBOX_WITH_STATISTICS
8787	switch (uVector)
8788	{
8789	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
8790	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
8791	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
8792	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8793	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
8794	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
8795	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8796	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
8797	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
8798	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
8799	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
8800	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
8801	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
8802	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
8803	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
8804	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
8805	default:
8806	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8807	break;
8808	}
8809	#endif
8810
8811	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8812	return VINF_SUCCESS;
8813	}
8814
8815
8816	/**
8817	* \#VMEXIT handler for software interrupt (INTn). Conditional \#VMEXIT (debug).
8818	*/
8819	HMSVM_EXIT_DECL hmR0SvmExitSwInt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8820	{
8821	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8822	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8823
8824	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8825	SVMEVENT Event;
8826	Event.u = 0;
8827	Event.n.u1Valid = 1;
8828	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
8829	Event.n.u8Vector = pVmcb->ctrl.u64ExitInfo1 & 0xff;
8830	Log4Func(("uVector=%#x\n", Event.n.u8Vector));
8831	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8832	return VINF_SUCCESS;
8833	}
8834
8835
8836	/**
8837	* Generic exit handler that interprets the current instruction
8838	*
8839	* Useful exit that only gets triggered by dtrace and the debugger. Caller does
8840	* the exit logging, and this function does the rest.
8841	*/
8842	static VBOXSTRICTRC hmR0SvmExitInterpretInstruction(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient,
8843	uint64_t fExtraImport, uint64_t fHmChanged)
8844	{
8845	#if 1
8846	RT_NOREF(pSvmTransient);
8847	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fExtraImport);
8848	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
8849	if (rcStrict == VINF_SUCCESS)
8850	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RIP);
8851	else
8852	{
8853	Log4Func(("IEMExecOne -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
8854	if (rcStrict == VINF_IEM_RAISED_XCPT)
8855	{
8856	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK \| fHmChanged);
8857	rcStrict = VINF_SUCCESS;
8858	}
8859	else
8860	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged);
8861	}
8862	return rcStrict;
8863	#else
8864	RT_NOREF(pVCpu, pSvmTransient, fExtraImport, fHmChanged);
8865	return VINF_EM_RAW_EMULATE_INSTR;
8866	#endif
8867	}
8868
8869
8870	/**
8871	* \#VMEXIT handler for STR. Conditional \#VMEXIT (debug).
8872	*/
8873	HMSVM_EXIT_DECL hmR0SvmExitTrRead(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8874	{
8875	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8876	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8877	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR, 0);
8878	}
8879
8880
8881	/**
8882	* \#VMEXIT handler for LTR. Conditional \#VMEXIT (OS/2 TLB workaround, debug).
8883	*/
8884	HMSVM_EXIT_DECL hmR0SvmExitTrWrite(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8885	{
8886	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8887
8888	/* Workaround for lack of TLB flushing in OS/2 when returning to protected
8889	mode after a real mode call (like a BIOS call). See ticketref:20625
8890	comment 14. */
8891	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8892	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
8893	{
8894	Log4Func(("%04x:%08RX64 TLB flush\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8895	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
8896	}
8897	else
8898	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8899
8900	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR, HM_CHANGED_GUEST_TR);
8901	}
8902
8903
8904	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8905	/**
8906	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
8907	*/
8908	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8909	{
8910	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8911
8912	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8913	Assert(pVmcb);
8914	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
8915
8916	VBOXSTRICTRC rcStrict;
8917	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8918	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8919	if (fSupportsNextRipSave)
8920	{
8921	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8922	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8923	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
8924	}
8925	else
8926	{
8927	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8928	rcStrict = IEMExecOne(pVCpu);
8929	}
8930
8931	if (rcStrict == VINF_SUCCESS)
8932	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8933	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8934	{
8935	rcStrict = VINF_SUCCESS;
8936	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8937	}
8938	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8939	return rcStrict;
8940	}
8941
8942
8943	/**
8944	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
8945	*/
8946	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8947	{
8948	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8949
8950	/*
8951	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
8952	* we only intercept STGI when events are pending for GIF to become 1.
8953	*/
8954	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8955	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
8956	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
8957
8958	VBOXSTRICTRC rcStrict;
8959	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8960	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8961	if (fSupportsNextRipSave)
8962	{
8963	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8964	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8965	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
8966	}
8967	else
8968	{
8969	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8970	rcStrict = IEMExecOne(pVCpu);
8971	}
8972
8973	if (rcStrict == VINF_SUCCESS)
8974	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8975	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8976	{
8977	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8978	rcStrict = VINF_SUCCESS;
8979	}
8980	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8981	return rcStrict;
8982	}
8983
8984
8985	/**
8986	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
8987	*/
8988	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8989	{
8990	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8991
8992	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8993	Assert(pVmcb);
8994	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
8995
8996	VBOXSTRICTRC rcStrict;
8997	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8998	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
8999	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
9000	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
9001	if (fSupportsNextRipSave)
9002	{
9003	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
9004	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9005	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
9006	}
9007	else
9008	{
9009	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
9010	rcStrict = IEMExecOne(pVCpu);
9011	}
9012
9013	if (rcStrict == VINF_SUCCESS)
9014	{
9015	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
9016	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
9017	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
9018	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
9019	}
9020	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9021	{
9022	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9023	rcStrict = VINF_SUCCESS;
9024	}
9025	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9026	return rcStrict;
9027	}
9028
9029
9030	/**
9031	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
9032	*/
9033	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9034	{
9035	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9036
9037	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9038	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9039
9040	VBOXSTRICTRC rcStrict;
9041	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9042	if (fSupportsNextRipSave)
9043	{
9044	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9045	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9046	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
9047	}
9048	else
9049	{
9050	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9051	rcStrict = IEMExecOne(pVCpu);
9052	}
9053
9054	if (rcStrict == VINF_IEM_RAISED_XCPT)
9055	{
9056	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9057	rcStrict = VINF_SUCCESS;
9058	}
9059	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9060	return rcStrict;
9061	}
9062
9063
9064	/**
9065	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
9066	*/
9067	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9068	{
9069	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9070
9071	VBOXSTRICTRC rcStrict;
9072	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9073	if (fSupportsNextRipSave)
9074	{
9075	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9076	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9077	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9078	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
9079	}
9080	else
9081	{
9082	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9083	rcStrict = IEMExecOne(pVCpu);
9084	}
9085
9086	if (rcStrict == VINF_IEM_RAISED_XCPT)
9087	{
9088	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9089	rcStrict = VINF_SUCCESS;
9090	}
9091	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9092	return rcStrict;
9093	}
9094
9095
9096	/**
9097	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
9098	*/
9099	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9100	{
9101	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9102	/* We shall import the entire state here, just in case we enter and continue execution of
9103	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
9104	could lose lose part of CPU state. */
9105	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
9106
9107	VBOXSTRICTRC rcStrict;
9108	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9109	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
9110	if (fSupportsNextRipSave)
9111	{
9112	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9113	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9114	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
9115	}
9116	else
9117	{
9118	/* We use IEMExecOneBypass() here as it suppresses attempt to continue emulating any
9119	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
9120	instruction itself, see @bugref{7243#c126} */
9121	rcStrict = IEMExecOneBypass(pVCpu);
9122	}
9123	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
9124
9125	if (rcStrict == VINF_SUCCESS)
9126	{
9127	rcStrict = VINF_SVM_VMRUN;
9128	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
9129	}
9130	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9131	{
9132	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9133	rcStrict = VINF_SUCCESS;
9134	}
9135	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9136	return rcStrict;
9137	}
9138
9139
9140	/**
9141	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
9142	* Unconditional \#VMEXIT.
9143	*/
9144	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9145	{
9146	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9147	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9148
9149	if (pVCpu->hm.s.Event.fPending)
9150	{
9151	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
9152	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9153	}
9154
9155	hmR0SvmSetPendingXcptDB(pVCpu);
9156	return VINF_SUCCESS;
9157	}
9158
9159
9160	/**
9161	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
9162	* Conditional \#VMEXIT.
9163	*/
9164	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9165	{
9166	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9167	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9168
9169	SVMEVENT Event;
9170	Event.u = 0;
9171	Event.n.u1Valid = 1;
9172	Event.n.u3Type = SVM_EVENT_EXCEPTION;
9173	Event.n.u8Vector = X86_XCPT_BP;
9174	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
9175	return VINF_SUCCESS;
9176	}
9177	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
9178
9179	/** @} */
9180

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp

Download in other formats: