VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 62229

Last change on this file since 62229 was 62106, checked in by vboxsync, 9 years ago

hmR0VmxIs32BitSwitcherSafe: Use return, it's one of the major perks implementing a predicate function as a function. Corrected docs.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 582.2 KB
Line 
1/* $Id: HMVMXR0.cpp 62106 2016-07-07 14:24:52Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#ifdef VBOX_WITH_NEW_APIC
38# include <VBox/vmm/apic.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_SWAP_FPU_STATE
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name Updated-guest-state flags.
71 * @{ */
72#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93 | HMVMX_UPDATED_GUEST_RSP \
94 | HMVMX_UPDATED_GUEST_RFLAGS \
95 | HMVMX_UPDATED_GUEST_CR0 \
96 | HMVMX_UPDATED_GUEST_CR3 \
97 | HMVMX_UPDATED_GUEST_CR4 \
98 | HMVMX_UPDATED_GUEST_GDTR \
99 | HMVMX_UPDATED_GUEST_IDTR \
100 | HMVMX_UPDATED_GUEST_LDTR \
101 | HMVMX_UPDATED_GUEST_TR \
102 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103 | HMVMX_UPDATED_GUEST_DEBUG \
104 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
109 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110 | HMVMX_UPDATED_GUEST_INTR_STATE \
111 | HMVMX_UPDATED_GUEST_APIC_STATE)
112/** @} */
113
114/** @name
115 * Flags to skip redundant reads of some common VMCS fields that are not part of
116 * the guest-CPU state but are in the transient structure.
117 */
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually except:
143 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145 * due to bugs in Intel CPUs.
146 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
147 * support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*********************************************************************************************************************************
198* Structures and Typedefs *
199*********************************************************************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG fEFlags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit code qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u7Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns Strict VBox status code (i.e. informational status codes too).
323 * @param pVCpu The cross context virtual CPU structure.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
345 * out-of-sync. Make sure to update the required
346 * fields before using them.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368 bool fStepping, uint32_t *puIntState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381
382/** @name VM-exit handlers.
383 * @{
384 */
385static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397static FNVMXEXITHANDLER hmR0VmxExitHlt;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413static FNVMXEXITHANDLER hmR0VmxExitMwait;
414static FNVMXEXITHANDLER hmR0VmxExitMtf;
415static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416static FNVMXEXITHANDLER hmR0VmxExitPause;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443/*********************************************************************************************************************************
444* Global Variables *
445*********************************************************************************************************************************/
446#ifdef HMVMX_USE_FUNCTION_TABLE
447
448/**
449 * VMX_EXIT dispatch table.
450 */
451static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452{
453 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493 /* 40 UNDEFINED */ hmR0VmxExitPause,
494 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
515 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518};
519#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521#ifdef VBOX_STRICT
522static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523{
524 /* 0 */ "(Not Used)",
525 /* 1 */ "VMCALL executed in VMX root operation.",
526 /* 2 */ "VMCLEAR with invalid physical address.",
527 /* 3 */ "VMCLEAR with VMXON pointer.",
528 /* 4 */ "VMLAUNCH with non-clear VMCS.",
529 /* 5 */ "VMRESUME with non-launched VMCS.",
530 /* 6 */ "VMRESUME after VMXOFF",
531 /* 7 */ "VM-entry with invalid control fields.",
532 /* 8 */ "VM-entry with invalid host state fields.",
533 /* 9 */ "VMPTRLD with invalid physical address.",
534 /* 10 */ "VMPTRLD with VMXON pointer.",
535 /* 11 */ "VMPTRLD with incorrect revision identifier.",
536 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537 /* 13 */ "VMWRITE to read-only VMCS component.",
538 /* 14 */ "(Not Used)",
539 /* 15 */ "VMXON executed in VMX root operation.",
540 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541 /* 17 */ "VM-entry with non-launched executing VMCS.",
542 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543 /* 19 */ "VMCALL with non-clear VMCS.",
544 /* 20 */ "VMCALL with invalid VM-exit control fields.",
545 /* 21 */ "(Not Used)",
546 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548 /* 24 */ "VMCALL with invalid SMM-monitor features.",
549 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550 /* 26 */ "VM-entry with events blocked by MOV SS.",
551 /* 27 */ "(Not Used)",
552 /* 28 */ "Invalid operand to INVEPT/INVVPID."
553};
554#endif /* VBOX_STRICT */
555
556
557
558/**
559 * Updates the VM's last error record.
560 *
561 * If there was a VMX instruction error, reads the error data from the VMCS and
562 * updates VCPU's last error record as well.
563 *
564 * @param pVM The cross context VM structure.
565 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
566 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567 * VERR_VMX_INVALID_VMCS_FIELD.
568 * @param rc The error code.
569 */
570static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571{
572 AssertPtr(pVM);
573 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574 || rc == VERR_VMX_UNABLE_TO_START_VM)
575 {
576 AssertPtrReturnVoid(pVCpu);
577 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578 }
579 pVM->hm.s.lLastError = rc;
580}
581
582
583/**
584 * Reads the VM-entry interruption-information field from the VMCS into the VMX
585 * transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit interruption-information field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 */
641DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642{
643 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644 {
645 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646 AssertRCReturn(rc, rc);
647 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648 }
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-exit interruption error code from the VMCS into the VMX
655 * transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 */
660DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661{
662 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663 {
664 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665 AssertRCReturn(rc, rc);
666 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667 }
668 return VINF_SUCCESS;
669}
670
671
672/**
673 * Reads the VM-exit instruction length field from the VMCS into the VMX
674 * transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVmxTransient Pointer to the VMX transient structure.
678 */
679DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680{
681 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682 {
683 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684 AssertRCReturn(rc, rc);
685 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686 }
687 return VINF_SUCCESS;
688}
689
690
691/**
692 * Reads the VM-exit instruction-information field from the VMCS into
693 * the VMX transient structure.
694 *
695 * @returns VBox status code.
696 * @param pVmxTransient Pointer to the VMX transient structure.
697 */
698DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699{
700 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701 {
702 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703 AssertRCReturn(rc, rc);
704 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705 }
706 return VINF_SUCCESS;
707}
708
709
710/**
711 * Reads the exit code qualification from the VMCS into the VMX transient
712 * structure.
713 *
714 * @returns VBox status code.
715 * @param pVCpu The cross context virtual CPU structure of the
716 * calling EMT. (Required for the VMCS cache case.)
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722 {
723 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the IDT-vectoring information field from the VMCS into the VMX
733 * transient structure.
734 *
735 * @returns VBox status code.
736 * @param pVmxTransient Pointer to the VMX transient structure.
737 *
738 * @remarks No-long-jump zone!!!
739 */
740DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743 {
744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring error code from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 */
759DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760{
761 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762 {
763 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764 AssertRCReturn(rc, rc);
765 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766 }
767 return VINF_SUCCESS;
768}
769
770
771/**
772 * Enters VMX root mode operation on the current CPU.
773 *
774 * @returns VBox status code.
775 * @param pVM The cross context VM structure. Can be
776 * NULL, after a resume.
777 * @param HCPhysCpuPage Physical address of the VMXON region.
778 * @param pvCpuPage Pointer to the VMXON region.
779 */
780static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781{
782 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784 Assert(pvCpuPage);
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 if (pVM)
788 {
789 /* Write the VMCS revision dword to the VMXON region. */
790 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791 }
792
793 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794 RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796 /* Enable the VMX bit in CR4 if necessary. */
797 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
798
799 /* Enter VMX root mode. */
800 int rc = VMXEnable(HCPhysCpuPage);
801 if (RT_FAILURE(rc))
802 {
803 if (!(uOldCr4 & X86_CR4_VMXE))
804 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806 if (pVM)
807 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808 }
809
810 /* Restore interrupts. */
811 ASMSetFlags(fEFlags);
812 return rc;
813}
814
815
816/**
817 * Exits VMX root mode operation on the current CPU.
818 *
819 * @returns VBox status code.
820 */
821static int hmR0VmxLeaveRootMode(void)
822{
823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826 RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828 /* If we're for some reason not in VMX root mode, then don't leave it. */
829 RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831 int rc;
832 if (uHostCR4 & X86_CR4_VMXE)
833 {
834 /* Exit VMX root mode and clear the VMX bit in CR4. */
835 VMXDisable();
836 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837 rc = VINF_SUCCESS;
838 }
839 else
840 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842 /* Restore interrupts. */
843 ASMSetFlags(fEFlags);
844 return rc;
845}
846
847
848/**
849 * Allocates and maps one physically contiguous page. The allocated page is
850 * zero'd out. (Used by various VT-x structures).
851 *
852 * @returns IPRT status code.
853 * @param pMemObj Pointer to the ring-0 memory object.
854 * @param ppVirt Where to store the virtual address of the
855 * allocation.
856 * @param pHCPhys Where to store the physical address of the
857 * allocation.
858 */
859DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860{
861 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866 if (RT_FAILURE(rc))
867 return rc;
868 *ppVirt = RTR0MemObjAddress(*pMemObj);
869 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
870 ASMMemZero32(*ppVirt, PAGE_SIZE);
871 return VINF_SUCCESS;
872}
873
874
875/**
876 * Frees and unmaps an allocated physical page.
877 *
878 * @param pMemObj Pointer to the ring-0 memory object.
879 * @param ppVirt Where to re-initialize the virtual address of
880 * allocation as 0.
881 * @param pHCPhys Where to re-initialize the physical address of the
882 * allocation as 0.
883 */
884DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885{
886 AssertPtr(pMemObj);
887 AssertPtr(ppVirt);
888 AssertPtr(pHCPhys);
889 if (*pMemObj != NIL_RTR0MEMOBJ)
890 {
891 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
892 AssertRC(rc);
893 *pMemObj = NIL_RTR0MEMOBJ;
894 *ppVirt = 0;
895 *pHCPhys = 0;
896 }
897}
898
899
900/**
901 * Worker function to free VT-x related structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM The cross context VM structure.
905 */
906static void hmR0VmxStructsFree(PVM pVM)
907{
908 for (VMCPUID i = 0; i < pVM->cCpus; i++)
909 {
910 PVMCPU pVCpu = &pVM->aCpus[i];
911 AssertPtr(pVCpu);
912
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921 }
922
923 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926#endif
927}
928
929
930/**
931 * Worker function to allocate VT-x related VM structures.
932 *
933 * @returns IPRT status code.
934 * @param pVM The cross context VM structure.
935 */
936static int hmR0VmxStructsAlloc(PVM pVM)
937{
938 /*
939 * Initialize members up-front so we can cleanup properly on allocation failure.
940 */
941#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944 pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953#endif
954 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#endif
1516 return false;
1517}
1518
1519
1520/**
1521 * Saves a set of guest MSRs back into the guest-CPU context.
1522 *
1523 * @param pVCpu The cross context virtual CPU structure.
1524 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525 * out-of-sync. Make sure to update the required fields
1526 * before using them.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1536 {
1537 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538#if HC_ARCH_BITS == 64
1539 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1540 {
1541 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1542 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1543 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1544 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1545 }
1546#endif
1547 }
1548}
1549
1550
1551/**
1552 * Loads a set of guests MSRs to allow read/passthru to the guest.
1553 *
1554 * The name of this function is slightly confusing. This function does NOT
1555 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1556 * common prefix for functions dealing with "lazy restoration" of the shared
1557 * MSRs.
1558 *
1559 * @param pVCpu The cross context virtual CPU structure.
1560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1561 * out-of-sync. Make sure to update the required fields
1562 * before using them.
1563 *
1564 * @remarks No-long-jump zone!!!
1565 */
1566static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1567{
1568 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1569 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1570
1571#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1572 do { \
1573 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1574 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1575 else \
1576 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1577 } while (0)
1578
1579 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1581 {
1582#if HC_ARCH_BITS == 64
1583 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1584 {
1585 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1586 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1587 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1588 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1589 }
1590#endif
1591 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1592 }
1593
1594#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1595}
1596
1597
1598/**
1599 * Performs lazy restoration of the set of host MSRs if they were previously
1600 * loaded with guest MSR values.
1601 *
1602 * @param pVCpu The cross context virtual CPU structure.
1603 *
1604 * @remarks No-long-jump zone!!!
1605 * @remarks The guest MSRs should have been saved back into the guest-CPU
1606 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1607 */
1608static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1609{
1610 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1612
1613 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1614 {
1615 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1616#if HC_ARCH_BITS == 64
1617 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1618 {
1619 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1620 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1621 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1622 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1623 }
1624#endif
1625 }
1626 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1627}
1628
1629
1630/**
1631 * Verifies that our cached values of the VMCS controls are all
1632 * consistent with what's actually present in the VMCS.
1633 *
1634 * @returns VBox status code.
1635 * @param pVCpu The cross context virtual CPU structure.
1636 */
1637static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1638{
1639 uint32_t u32Val;
1640 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1643 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1648 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1654
1655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1656 AssertRCReturn(rc, rc);
1657 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1658 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1659
1660 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661 {
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1666 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1667 }
1668
1669 return VINF_SUCCESS;
1670}
1671
1672
1673#ifdef VBOX_STRICT
1674/**
1675 * Verifies that our cached host EFER value has not changed
1676 * since we cached it.
1677 *
1678 * @param pVCpu The cross context virtual CPU structure.
1679 */
1680static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1681{
1682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1685 {
1686 uint64_t u64Val;
1687 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1688 AssertRC(rc);
1689
1690 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1691 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1692 }
1693}
1694
1695
1696/**
1697 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1698 * VMCS are correct.
1699 *
1700 * @param pVCpu The cross context virtual CPU structure.
1701 */
1702static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1703{
1704 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1705
1706 /* Verify MSR counts in the VMCS are what we think it should be. */
1707 uint32_t cMsrs;
1708 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1718 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1719 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1720 {
1721 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1722 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1723 pGuestMsr->u32Msr, cMsrs));
1724
1725 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1726 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1727 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1728
1729 /* Verify that the permissions are as expected in the MSR bitmap. */
1730 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1731 {
1732 VMXMSREXITREAD enmRead;
1733 VMXMSREXITWRITE enmWrite;
1734 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1735 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1736 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1739 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1740 }
1741 else
1742 {
1743 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1744 pGuestMsr->u32Msr, cMsrs));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1746 pGuestMsr->u32Msr, cMsrs));
1747 }
1748 }
1749 }
1750}
1751#endif /* VBOX_STRICT */
1752
1753
1754/**
1755 * Flushes the TLB using EPT.
1756 *
1757 * @returns VBox status code.
1758 * @param pVCpu The cross context virtual CPU structure of the calling
1759 * EMT. Can be NULL depending on @a enmFlush.
1760 * @param enmFlush Type of flush.
1761 *
1762 * @remarks Caller is responsible for making sure this function is called only
1763 * when NestedPaging is supported and providing @a enmFlush that is
1764 * supported by the CPU.
1765 * @remarks Can be called with interrupts disabled.
1766 */
1767static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1768{
1769 uint64_t au64Descriptor[2];
1770 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1771 au64Descriptor[0] = 0;
1772 else
1773 {
1774 Assert(pVCpu);
1775 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1776 }
1777 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1778
1779 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1780 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1781 rc));
1782 if ( RT_SUCCESS(rc)
1783 && pVCpu)
1784 {
1785 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1786 }
1787}
1788
1789
1790/**
1791 * Flushes the TLB using VPID.
1792 *
1793 * @returns VBox status code.
1794 * @param pVM The cross context VM structure.
1795 * @param pVCpu The cross context virtual CPU structure of the calling
1796 * EMT. Can be NULL depending on @a enmFlush.
1797 * @param enmFlush Type of flush.
1798 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1799 * on @a enmFlush).
1800 *
1801 * @remarks Can be called with interrupts disabled.
1802 */
1803static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804{
1805 NOREF(pVM);
1806 AssertPtr(pVM);
1807 Assert(pVM->hm.s.vmx.fVpid);
1808
1809 uint64_t au64Descriptor[2];
1810 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1811 {
1812 au64Descriptor[0] = 0;
1813 au64Descriptor[1] = 0;
1814 }
1815 else
1816 {
1817 AssertPtr(pVCpu);
1818 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1819 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1820 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1821 au64Descriptor[1] = GCPtr;
1822 }
1823
1824 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1825 AssertMsg(rc == VINF_SUCCESS,
1826 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1827 if ( RT_SUCCESS(rc)
1828 && pVCpu)
1829 {
1830 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1831 }
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by guest virtual address. Only relevant for
1837 * EPT/VPID, otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM The cross context VM structure.
1841 * @param pVCpu The cross context virtual CPU structure.
1842 * @param GCVirt Guest virtual address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1845{
1846 AssertPtr(pVM);
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1855 * See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1858 * function maybe called in a loop with individual addresses.
1859 */
1860 if (pVM->hm.s.vmx.fVpid)
1861 {
1862 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1863 {
1864 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1880 * otherwise there is nothing really to invalidate.
1881 *
1882 * @returns VBox status code.
1883 * @param pVM The cross context VM structure.
1884 * @param pVCpu The cross context virtual CPU structure.
1885 * @param GCPhys Guest physical address of the page to invalidate.
1886 */
1887VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1888{
1889 NOREF(pVM); NOREF(GCPhys);
1890 LogFlowFunc(("%RGp\n", GCPhys));
1891
1892 /*
1893 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1894 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1895 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1896 */
1897 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1899 return VINF_SUCCESS;
1900}
1901
1902
1903/**
1904 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1905 * case where neither EPT nor VPID is supported by the CPU.
1906 *
1907 * @param pVM The cross context VM structure.
1908 * @param pVCpu The cross context virtual CPU structure.
1909 * @param pCpu Pointer to the global HM struct.
1910 *
1911 * @remarks Called with interrupts disabled.
1912 */
1913static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914{
1915 AssertPtr(pVCpu);
1916 AssertPtr(pCpu);
1917 NOREF(pVM);
1918
1919 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924 pVCpu->hm.s.fForceTLBFlush = false;
1925 return;
1926}
1927
1928
1929/**
1930 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931 *
1932 * @param pVM The cross context VM structure.
1933 * @param pVCpu The cross context virtual CPU structure.
1934 * @param pCpu Pointer to the global HM CPU struct.
1935 * @remarks All references to "ASID" in this function pertains to "VPID" in
1936 * Intel's nomenclature. The reason is, to avoid confusion in compare
1937 * statements since the host-CPU copies are named "ASID".
1938 *
1939 * @remarks Called with interrupts disabled.
1940 */
1941static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942{
1943#ifdef VBOX_WITH_STATISTICS
1944 bool fTlbFlushed = false;
1945# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947 if (!fTlbFlushed) \
1948 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949 } while (0)
1950#else
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953#endif
1954
1955 AssertPtr(pVM);
1956 AssertPtr(pCpu);
1957 AssertPtr(pVCpu);
1958 Assert(pCpu->idCpu != NIL_RTCPUID);
1959
1960 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964 /*
1965 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1966 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1967 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1968 */
1969 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971 {
1972 ++pCpu->uCurrentAsid;
1973 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974 {
1975 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978 }
1979
1980 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984 /*
1985 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987 */
1988 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1989 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990 HMVMX_SET_TAGGED_TLB_FLUSHED();
1991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1992 }
1993
1994 /* Check for explicit TLB flushes. */
1995 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1996 {
1997 /*
1998 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1999 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2000 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2001 * but not guest-physical mappings.
2002 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2003 */
2004 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006 HMVMX_SET_TAGGED_TLB_FLUSHED();
2007 }
2008
2009 pVCpu->hm.s.fForceTLBFlush = false;
2010 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022 /* Update VMCS with the VPID. */
2023 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024 AssertRC(rc);
2025
2026#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027}
2028
2029
2030/**
2031 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032 *
2033 * @returns VBox status code.
2034 * @param pVM The cross context VM structure.
2035 * @param pVCpu The cross context virtual CPU structure.
2036 * @param pCpu Pointer to the global HM CPU struct.
2037 *
2038 * @remarks Called with interrupts disabled.
2039 */
2040static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2041{
2042 AssertPtr(pVM);
2043 AssertPtr(pVCpu);
2044 AssertPtr(pCpu);
2045 Assert(pCpu->idCpu != NIL_RTCPUID);
2046 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2047 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2048
2049 /*
2050 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2051 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2052 */
2053 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2054 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2055 {
2056 pVCpu->hm.s.fForceTLBFlush = true;
2057 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2058 }
2059
2060 /* Check for explicit TLB flushes. */
2061 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2062 {
2063 pVCpu->hm.s.fForceTLBFlush = true;
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2065 }
2066
2067 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2068 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2069
2070 if (pVCpu->hm.s.fForceTLBFlush)
2071 {
2072 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2073 pVCpu->hm.s.fForceTLBFlush = false;
2074 }
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM The cross context VM structure.
2083 * @param pVCpu The cross context virtual CPU structure.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151
2152 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2153 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2154 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2155 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2156 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2158 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2159
2160 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2161 AssertRC(rc);
2162}
2163
2164
2165/**
2166 * Flushes the guest TLB entry based on CPU capabilities.
2167 *
2168 * @param pVCpu The cross context virtual CPU structure.
2169 * @param pCpu Pointer to the global HM CPU struct.
2170 */
2171DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2172{
2173#ifdef HMVMX_ALWAYS_FLUSH_TLB
2174 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2175#endif
2176 PVM pVM = pVCpu->CTX_SUFF(pVM);
2177 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2178 {
2179 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2180 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2181 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2182 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2183 default:
2184 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2185 break;
2186 }
2187
2188 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2189}
2190
2191
2192/**
2193 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2194 * TLB entries from the host TLB before VM-entry.
2195 *
2196 * @returns VBox status code.
2197 * @param pVM The cross context VM structure.
2198 */
2199static int hmR0VmxSetupTaggedTlb(PVM pVM)
2200{
2201 /*
2202 * Determine optimal flush type for Nested Paging.
2203 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2204 * guest execution (see hmR3InitFinalizeR0()).
2205 */
2206 if (pVM->hm.s.fNestedPaging)
2207 {
2208 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2209 {
2210 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2212 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221
2222 /* Make sure the write-back cacheable memory type for EPT is supported. */
2223 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2224 {
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* EPT requires a page-walk length of 4. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237 }
2238 else
2239 {
2240 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246
2247 /*
2248 * Determine optimal flush type for VPID.
2249 */
2250 if (pVM->hm.s.vmx.fVpid)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2262 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2264 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2266 pVM->hm.s.vmx.fVpid = false;
2267 }
2268 }
2269 else
2270 {
2271 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2272 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277
2278 /*
2279 * Setup the handler for flushing tagged-TLBs.
2280 */
2281 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2282 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2283 else if (pVM->hm.s.fNestedPaging)
2284 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2285 else if (pVM->hm.s.vmx.fVpid)
2286 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2287 else
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2289 return VINF_SUCCESS;
2290}
2291
2292
2293/**
2294 * Sets up pin-based VM-execution controls in the VMCS.
2295 *
2296 * @returns VBox status code.
2297 * @param pVM The cross context VM structure.
2298 * @param pVCpu The cross context virtual CPU structure.
2299 */
2300static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2301{
2302 AssertPtr(pVM);
2303 AssertPtr(pVCpu);
2304
2305 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2306 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2307
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2309 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2310
2311 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2312 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2313
2314 /* Enable the VMX preemption timer. */
2315 if (pVM->hm.s.vmx.fUsePreemptTimer)
2316 {
2317 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2319 }
2320
2321#ifdef VBOX_WITH_NEW_APIC
2322#if 0
2323 /* Enable posted-interrupt processing. */
2324 if (pVM->hm.s.fPostedIntrs)
2325 {
2326 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329 }
2330#endif
2331#endif
2332
2333 if ((val & zap) != val)
2334 {
2335 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2336 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2337 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2338 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2339 }
2340
2341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2342 AssertRCReturn(rc, rc);
2343
2344 pVCpu->hm.s.vmx.u32PinCtls = val;
2345 return rc;
2346}
2347
2348
2349/**
2350 * Sets up processor-based VM-execution controls in the VMCS.
2351 *
2352 * @returns VBox status code.
2353 * @param pVM The cross context VM structure.
2354 * @param pVCpu The cross context virtual CPU structure.
2355 */
2356static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2357{
2358 AssertPtr(pVM);
2359 AssertPtr(pVCpu);
2360
2361 int rc = VERR_INTERNAL_ERROR_5;
2362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2364
2365 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2368 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2369 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2372
2373 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2374 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2375 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2376 {
2377 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2378 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2379 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2380 }
2381
2382 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2383 if (!pVM->hm.s.fNestedPaging)
2384 {
2385 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2386 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2387 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2389 }
2390
2391 /* Use TPR shadowing if supported by the CPU. */
2392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2393 {
2394 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2397 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2398 AssertRCReturn(rc, rc);
2399
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2401 /* CR8 writes cause a VM-exit based on TPR threshold. */
2402 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2403 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2404 }
2405 else
2406 {
2407 /*
2408 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2409 * Set this control only for 64-bit guests.
2410 */
2411 if (pVM->hm.s.fAllow64BitGuests)
2412 {
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2414 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2415 }
2416 }
2417
2418 /* Use MSR-bitmaps if supported by the CPU. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2420 {
2421 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2422
2423 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2424 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2425 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2426 AssertRCReturn(rc, rc);
2427
2428 /*
2429 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2430 * automatically using dedicated fields in the VMCS.
2431 */
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2434 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2435 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437
2438#if HC_ARCH_BITS == 64
2439 /*
2440 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2441 */
2442 if (pVM->hm.s.fAllow64BitGuests)
2443 {
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 }
2449#endif
2450 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2451 }
2452
2453 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2454 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2455 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2456
2457 if ((val & zap) != val)
2458 {
2459 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2460 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2461 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2462 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463 }
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2466 AssertRCReturn(rc, rc);
2467
2468 pVCpu->hm.s.vmx.u32ProcCtls = val;
2469
2470 /*
2471 * Secondary processor-based VM-execution controls.
2472 */
2473 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2474 {
2475 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2476 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2477
2478 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2479 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2480
2481 if (pVM->hm.s.fNestedPaging)
2482 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2483 else
2484 {
2485 /*
2486 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2487 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2488 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2489 */
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2492 }
2493
2494 if (pVM->hm.s.vmx.fVpid)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2496
2497 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2498 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2499
2500#ifdef VBOX_WITH_NEW_APIC
2501#if 0
2502 if (pVM->hm.s.fVirtApicRegs)
2503 {
2504 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2506
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2509 }
2510#endif
2511#endif
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2529 && pVM->hm.s.vmx.cPleGapTicks
2530 && pVM->hm.s.vmx.cPleWindowTicks)
2531 {
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2535 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2536 AssertRCReturn(rc, rc);
2537 }
2538
2539 if ((val & zap) != val)
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2542 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2548 AssertRCReturn(rc, rc);
2549
2550 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2551 }
2552 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553 {
2554 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2555 "available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 return VINF_SUCCESS;
2561}
2562
2563
2564/**
2565 * Sets up miscellaneous (everything other than Pin & Processor-based
2566 * VM-execution) control fields in the VMCS.
2567 *
2568 * @returns VBox status code.
2569 * @param pVM The cross context VM structure.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2573{
2574 NOREF(pVM);
2575 AssertPtr(pVM);
2576 AssertPtr(pVCpu);
2577
2578 int rc = VERR_GENERAL_FAILURE;
2579
2580 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581#if 0
2582 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586 /*
2587 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589 * We thus use the exception bitmap to control it rather than use both.
2590 */
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594 /** @todo Explore possibility of using IO-bitmaps. */
2595 /* All IO & IOIO instructions cause VM-exits. */
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2598
2599 /* Initialize the MSR-bitmap area. */
2600 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2601 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2603 AssertRCReturn(rc, rc);
2604#endif
2605
2606 /* Setup MSR auto-load/store area. */
2607 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 AssertRCReturn(rc, rc);
2617
2618 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2620 AssertRCReturn(rc, rc);
2621
2622 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2623#if 0
2624 /* Setup debug controls */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2626 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2627 AssertRCReturn(rc, rc);
2628#endif
2629
2630 return rc;
2631}
2632
2633
2634/**
2635 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2636 *
2637 * @returns VBox status code.
2638 * @param pVM The cross context VM structure.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2642{
2643 AssertPtr(pVM);
2644 AssertPtr(pVCpu);
2645
2646 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2647
2648 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2649
2650 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2652
2653 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2654 and writes, and because recursive #DBs can cause the CPU hang, we must always
2655 intercept #DB. */
2656 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2657
2658 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2659 if (!pVM->hm.s.fNestedPaging)
2660 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2661
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2664 AssertRCReturn(rc, rc);
2665 return rc;
2666}
2667
2668
2669/**
2670 * Sets up the initial guest-state mask. The guest-state mask is consulted
2671 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2672 * for the nested virtualization case (as it would cause a VM-exit).
2673 *
2674 * @param pVCpu The cross context virtual CPU structure.
2675 */
2676static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2677{
2678 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2679 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2740 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791 /* Set revision dword at the beginning of the VMCS structure. */
2792 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 /* Load this VMCS as the current VMCS. */
2800 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824#if HC_ARCH_BITS == 32
2825 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828#endif
2829
2830 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838 }
2839
2840 return VINF_SUCCESS;
2841}
2842
2843
2844/**
2845 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846 * the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVM The cross context VM structure.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853{
2854 NOREF(pVM); NOREF(pVCpu);
2855
2856 RTCCUINTREG uReg = ASMGetCR0();
2857 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858 AssertRCReturn(rc, rc);
2859
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 AssertRCReturn(rc, rc);
2863
2864 uReg = ASMGetCR4();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866 AssertRCReturn(rc, rc);
2867 return rc;
2868}
2869
2870
2871#if HC_ARCH_BITS == 64
2872/**
2873 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874 * requirements. See hmR0VmxSaveHostSegmentRegs().
2875 */
2876# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2878 { \
2879 bool fValidSelector = true; \
2880 if ((selValue) & X86_SEL_LDT) \
2881 { \
2882 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884 } \
2885 if (fValidSelector) \
2886 { \
2887 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2888 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889 } \
2890 (selValue) = 0; \
2891 }
2892#endif
2893
2894
2895/**
2896 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897 * the host-state area in the VMCS.
2898 *
2899 * @returns VBox status code.
2900 * @param pVM The cross context VM structure.
2901 * @param pVCpu The cross context virtual CPU structure.
2902 */
2903DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904{
2905 int rc = VERR_INTERNAL_ERROR_5;
2906
2907#if HC_ARCH_BITS == 64
2908 /*
2909 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2910 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2911 *
2912 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2913 * Was observed booting Solaris10u10 32-bit guest.
2914 */
2915 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2916 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2917 {
2918 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2919 pVCpu->idCpu));
2920 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2921 }
2922 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2923#endif
2924
2925 /*
2926 * Host DS, ES, FS and GS segment registers.
2927 */
2928#if HC_ARCH_BITS == 64
2929 RTSEL uSelDS = ASMGetDS();
2930 RTSEL uSelES = ASMGetES();
2931 RTSEL uSelFS = ASMGetFS();
2932 RTSEL uSelGS = ASMGetGS();
2933#else
2934 RTSEL uSelDS = 0;
2935 RTSEL uSelES = 0;
2936 RTSEL uSelFS = 0;
2937 RTSEL uSelGS = 0;
2938#endif
2939
2940 /*
2941 * Host CS and SS segment registers.
2942 */
2943 RTSEL uSelCS = ASMGetCS();
2944 RTSEL uSelSS = ASMGetSS();
2945
2946 /*
2947 * Host TR segment register.
2948 */
2949 RTSEL uSelTR = ASMGetTR();
2950
2951#if HC_ARCH_BITS == 64
2952 /*
2953 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2954 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2955 */
2956 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2957 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2958 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2959 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2960# undef VMXLOCAL_ADJUST_HOST_SEG
2961#endif
2962
2963 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2964 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2965 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2966 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2967 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2968 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2969 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2970 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2971 Assert(uSelCS);
2972 Assert(uSelTR);
2973
2974 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2975#if 0
2976 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2977 Assert(uSelSS != 0);
2978#endif
2979
2980 /* Write these host selector fields into the host-state area in the VMCS. */
2981 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2983#if HC_ARCH_BITS == 64
2984 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2985 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2986 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2987 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2988#else
2989 NOREF(uSelDS);
2990 NOREF(uSelES);
2991 NOREF(uSelFS);
2992 NOREF(uSelGS);
2993#endif
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2995 AssertRCReturn(rc, rc);
2996
2997 /*
2998 * Host GDTR and IDTR.
2999 */
3000 RTGDTR Gdtr;
3001 RTIDTR Idtr;
3002 RT_ZERO(Gdtr);
3003 RT_ZERO(Idtr);
3004 ASMGetGDTR(&Gdtr);
3005 ASMGetIDTR(&Idtr);
3006 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3007 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3008 AssertRCReturn(rc, rc);
3009
3010#if HC_ARCH_BITS == 64
3011 /*
3012 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3013 * maximum limit (0xffff) on every VM-exit.
3014 */
3015 if (Gdtr.cbGdt != 0xffff)
3016 {
3017 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3018 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3019 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3020 }
3021
3022 /*
3023 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3024 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3025 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3026 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3027 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3028 * hosts where we are pretty sure it won't cause trouble.
3029 */
3030# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3031 if (Idtr.cbIdt < 0x0fff)
3032# else
3033 if (Idtr.cbIdt != 0xffff)
3034# endif
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3037 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3039 }
3040#endif
3041
3042 /*
3043 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3044 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3045 */
3046 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3047 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3048 VERR_VMX_INVALID_HOST_STATE);
3049
3050 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3051#if HC_ARCH_BITS == 64
3052 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3053
3054 /*
3055 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3056 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3057 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3058 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3059 *
3060 * [1] See Intel spec. 3.5 "System Descriptor Types".
3061 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3062 */
3063 Assert(pDesc->System.u4Type == 11);
3064 if ( pDesc->System.u16LimitLow != 0x67
3065 || pDesc->System.u4LimitHigh)
3066 {
3067 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3068 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3069 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3071 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3072
3073 /* Store the GDTR here as we need it while restoring TR. */
3074 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3075 }
3076#else
3077 NOREF(pVM);
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return rc;
3100}
3101
3102
3103/**
3104 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3105 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3106 * the host after every successful VM-exit.
3107 *
3108 * @returns VBox status code.
3109 * @param pVM The cross context VM structure.
3110 * @param pVCpu The cross context virtual CPU structure.
3111 *
3112 * @remarks No-long-jump zone!!!
3113 */
3114DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3115{
3116 NOREF(pVM);
3117
3118 AssertPtr(pVCpu);
3119 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3120
3121 /*
3122 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3123 * rather than swapping them on every VM-entry.
3124 */
3125 hmR0VmxLazySaveHostMsrs(pVCpu);
3126
3127 /*
3128 * Host Sysenter MSRs.
3129 */
3130 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3131#if HC_ARCH_BITS == 32
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3133 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3134#else
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3136 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3137#endif
3138 AssertRCReturn(rc, rc);
3139
3140 /*
3141 * Host EFER MSR.
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3143 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3144 */
3145 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3146 {
3147 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3148 AssertRCReturn(rc, rc);
3149 }
3150
3151 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3152 * hmR0VmxLoadGuestExitCtls() !! */
3153
3154 return rc;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3163 * hmR0VMxLoadGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3168 * out-of-sync. Make sure to update the required fields
3169 * before using them.
3170 *
3171 * @remarks Requires EFER, CR4.
3172 * @remarks No-long-jump zone!!!
3173 */
3174static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3175{
3176#ifdef HMVMX_ALWAYS_SWAP_EFER
3177 return true;
3178#endif
3179
3180#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3181 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3182 if (CPUMIsGuestInLongMode(pVCpu))
3183 return false;
3184#endif
3185
3186 PVM pVM = pVCpu->CTX_SUFF(pVM);
3187 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3188 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3189
3190 /*
3191 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3192 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3193 */
3194 if ( CPUMIsGuestInLongMode(pVCpu)
3195 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3196 {
3197 return true;
3198 }
3199
3200 /*
3201 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3202 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3203 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3204 */
3205 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3206 && (pMixedCtx->cr0 & X86_CR0_PG)
3207 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3208 {
3209 /* Assert that host is PAE capable. */
3210 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3211 return true;
3212 }
3213
3214 /** @todo Check the latest Intel spec. for any other bits,
3215 * like SMEP/SMAP? */
3216 return false;
3217}
3218
3219
3220/**
3221 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3222 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3223 * controls".
3224 *
3225 * @returns VBox status code.
3226 * @param pVCpu The cross context virtual CPU structure.
3227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3228 * out-of-sync. Make sure to update the required fields
3229 * before using them.
3230 *
3231 * @remarks Requires EFER.
3232 * @remarks No-long-jump zone!!!
3233 */
3234DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3235{
3236 int rc = VINF_SUCCESS;
3237 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3238 {
3239 PVM pVM = pVCpu->CTX_SUFF(pVM);
3240 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3241 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3242
3243 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3244 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3245
3246 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3247 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3248 {
3249 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3250 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3251 }
3252 else
3253 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3254
3255 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3256 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3257 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3258 {
3259 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3260 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3261 }
3262
3263 /*
3264 * The following should -not- be set (since we're not in SMM mode):
3265 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3266 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3267 */
3268
3269 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3270 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3271
3272 if ((val & zap) != val)
3273 {
3274 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3275 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3276 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3277 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3278 }
3279
3280 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3281 AssertRCReturn(rc, rc);
3282
3283 pVCpu->hm.s.vmx.u32EntryCtls = val;
3284 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3285 }
3286 return rc;
3287}
3288
3289
3290/**
3291 * Sets up the VM-exit controls in the VMCS.
3292 *
3293 * @returns VBox status code.
3294 * @param pVCpu The cross context virtual CPU structure.
3295 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3296 * out-of-sync. Make sure to update the required fields
3297 * before using them.
3298 *
3299 * @remarks Requires EFER.
3300 */
3301DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3302{
3303 NOREF(pMixedCtx);
3304
3305 int rc = VINF_SUCCESS;
3306 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3307 {
3308 PVM pVM = pVCpu->CTX_SUFF(pVM);
3309 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3310 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3311
3312 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3313 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3314
3315 /*
3316 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3317 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3318 */
3319#if HC_ARCH_BITS == 64
3320 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3321 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3322#else
3323 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3324 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3325 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3326 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3327 {
3328 /* The switcher returns to long mode, EFER is managed by the switcher. */
3329 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3330 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3331 }
3332 else
3333 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3334#endif
3335
3336 /* If the newer VMCS fields for managing EFER exists, use it. */
3337 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3338 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3339 {
3340 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3341 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3342 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3343 }
3344
3345 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3346 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3347
3348 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3349 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3350 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3351
3352 if ( pVM->hm.s.vmx.fUsePreemptTimer
3353 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3354 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3355
3356 if ((val & zap) != val)
3357 {
3358 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3359 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3362 }
3363
3364 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3365 AssertRCReturn(rc, rc);
3366
3367 pVCpu->hm.s.vmx.u32ExitCtls = val;
3368 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3369 }
3370 return rc;
3371}
3372
3373
3374/**
3375 * Sets the TPR threshold in the VMCS.
3376 *
3377 * @returns VBox status code.
3378 * @param pVCpu The cross context virtual CPU structure.
3379 * @param u32TprThreshold The TPR threshold (task-priority class only).
3380 */
3381DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3382{
3383 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3384 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
3385 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3386}
3387
3388
3389/**
3390 * Loads the guest APIC and related state.
3391 *
3392 * @returns VBox status code.
3393 * @param pVCpu The cross context virtual CPU structure.
3394 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3395 * out-of-sync. Make sure to update the required fields
3396 * before using them.
3397 */
3398DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3399{
3400 NOREF(pMixedCtx);
3401
3402 int rc = VINF_SUCCESS;
3403 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3404 {
3405 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3418 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3419 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3420 * the interrupt when we VM-exit for other reasons.
3421 */
3422 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3423 uint32_t u32TprThreshold = 0;
3424 if (fPendingIntr)
3425 {
3426 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3427 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3428 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3429 if (u8PendingPriority <= u8TprPriority)
3430 u32TprThreshold = u8PendingPriority;
3431 else
3432 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3433 }
3434
3435 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3436 AssertRCReturn(rc, rc);
3437 }
3438
3439 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3440 }
3441 return rc;
3442}
3443
3444
3445/**
3446 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3447 *
3448 * @returns Guest's interruptibility-state.
3449 * @param pVCpu The cross context virtual CPU structure.
3450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3451 * out-of-sync. Make sure to update the required fields
3452 * before using them.
3453 *
3454 * @remarks No-long-jump zone!!!
3455 */
3456DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3457{
3458 /*
3459 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3460 */
3461 uint32_t uIntrState = 0;
3462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3463 {
3464 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3465 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3466 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3467 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pMixedCtx->eflags.Bits.u1IF)
3470 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3478 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3479 */
3480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3481 }
3482 }
3483
3484 /*
3485 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3486 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3487 * setting this would block host-NMIs and IRET will not clear the blocking.
3488 *
3489 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3490 */
3491 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3492 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3493 {
3494 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3495 }
3496
3497 return uIntrState;
3498}
3499
3500
3501/**
3502 * Loads the guest's interruptibility-state into the guest-state area in the
3503 * VMCS.
3504 *
3505 * @returns VBox status code.
3506 * @param pVCpu The cross context virtual CPU structure.
3507 * @param uIntrState The interruptibility-state to set.
3508 */
3509static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3510{
3511 NOREF(pVCpu);
3512 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3513 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3514 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3515 AssertRC(rc);
3516 return rc;
3517}
3518
3519
3520/**
3521 * Loads the exception intercepts required for guest execution in the VMCS.
3522 *
3523 * @returns VBox status code.
3524 * @param pVCpu The cross context virtual CPU structure.
3525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3526 * out-of-sync. Make sure to update the required fields
3527 * before using them.
3528 */
3529static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3530{
3531 NOREF(pMixedCtx);
3532 int rc = VINF_SUCCESS;
3533 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3534 {
3535 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3536 if (pVCpu->hm.s.fGIMTrapXcptUD)
3537 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3538#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3539 else
3540 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3541#endif
3542
3543 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3544 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3545
3546 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3547 AssertRCReturn(rc, rc);
3548
3549 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3550 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3551 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3552 }
3553 return rc;
3554}
3555
3556
3557/**
3558 * Loads the guest's RIP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu The cross context virtual CPU structure.
3562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3563 * out-of-sync. Make sure to update the required fields
3564 * before using them.
3565 *
3566 * @remarks No-long-jump zone!!!
3567 */
3568static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3569{
3570 int rc = VINF_SUCCESS;
3571 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3572 {
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3574 AssertRCReturn(rc, rc);
3575
3576 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3577 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3578 HMCPU_CF_VALUE(pVCpu)));
3579 }
3580 return rc;
3581}
3582
3583
3584/**
3585 * Loads the guest's RSP into the guest-state area in the VMCS.
3586 *
3587 * @returns VBox status code.
3588 * @param pVCpu The cross context virtual CPU structure.
3589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590 * out-of-sync. Make sure to update the required fields
3591 * before using them.
3592 *
3593 * @remarks No-long-jump zone!!!
3594 */
3595static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596{
3597 int rc = VINF_SUCCESS;
3598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3599 {
3600 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3601 AssertRCReturn(rc, rc);
3602
3603 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3604 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3605 }
3606 return rc;
3607}
3608
3609
3610/**
3611 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu The cross context virtual CPU structure.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 int rc = VINF_SUCCESS;
3624 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3625 {
3626 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3627 Let us assert it as such and use 32-bit VMWRITE. */
3628 Assert(!(pMixedCtx->rflags.u64 >> 32));
3629 X86EFLAGS Eflags = pMixedCtx->eflags;
3630 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3631 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3632 * These will never be cleared/set, unless some other part of the VMM
3633 * code is buggy - in which case we're better of finding and fixing
3634 * those bugs than hiding them. */
3635 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3636 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3637 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3638 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3639
3640 /*
3641 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3642 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3643 */
3644 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3645 {
3646 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3647 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3648 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3649 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3650 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3651 }
3652
3653 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3654 AssertRCReturn(rc, rc);
3655
3656 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3657 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3658 }
3659 return rc;
3660}
3661
3662
3663/**
3664 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3665 *
3666 * @returns VBox status code.
3667 * @param pVCpu The cross context virtual CPU structure.
3668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3669 * out-of-sync. Make sure to update the required fields
3670 * before using them.
3671 *
3672 * @remarks No-long-jump zone!!!
3673 */
3674DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3675{
3676 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3677 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3678 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3679 AssertRCReturn(rc, rc);
3680 return rc;
3681}
3682
3683
3684/**
3685 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3686 * CR0 is partially shared with the host and we have to consider the FPU bits.
3687 *
3688 * @returns VBox status code.
3689 * @param pVCpu The cross context virtual CPU structure.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 {
3777 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3778 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3779 }
3780 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3781
3782 if (fInterceptNM)
3783 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3784 else
3785 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3786
3787 if (fInterceptMF)
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3789 else
3790 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3791
3792 /* Additional intercepts for debugging, define these yourself explicitly. */
3793#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3794 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3795 | RT_BIT(X86_XCPT_BP)
3796 | RT_BIT(X86_XCPT_DE)
3797 | RT_BIT(X86_XCPT_NM)
3798 | RT_BIT(X86_XCPT_TS)
3799 | RT_BIT(X86_XCPT_UD)
3800 | RT_BIT(X86_XCPT_NP)
3801 | RT_BIT(X86_XCPT_SS)
3802 | RT_BIT(X86_XCPT_GP)
3803 | RT_BIT(X86_XCPT_PF)
3804 | RT_BIT(X86_XCPT_MF)
3805 ;
3806#elif defined(HMVMX_ALWAYS_TRAP_PF)
3807 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3808#endif
3809
3810 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3811
3812 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3813 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3814 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3815 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3816 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3817 else
3818 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3819
3820 u32GuestCR0 |= uSetCR0;
3821 u32GuestCR0 &= uZapCR0;
3822 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3823
3824 /* Write VT-x's view of the guest CR0 into the VMCS. */
3825 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3826 AssertRCReturn(rc, rc);
3827 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3828 uZapCR0));
3829
3830 /*
3831 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3832 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3833 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3834 */
3835 uint32_t u32CR0Mask = 0;
3836 u32CR0Mask = X86_CR0_PE
3837 | X86_CR0_NE
3838 | X86_CR0_WP
3839 | X86_CR0_PG
3840 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3841 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3842 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3843
3844 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3845 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3846 * and @bugref{6944}. */
3847#if 0
3848 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3849 u32CR0Mask &= ~X86_CR0_PE;
3850#endif
3851 if (pVM->hm.s.fNestedPaging)
3852 u32CR0Mask &= ~X86_CR0_WP;
3853
3854 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3855 if (fInterceptNM)
3856 {
3857 u32CR0Mask |= X86_CR0_TS
3858 | X86_CR0_MP;
3859 }
3860
3861 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3862 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3863 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3864 AssertRCReturn(rc, rc);
3865 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3866
3867 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3868 }
3869 return rc;
3870}
3871
3872
3873/**
3874 * Loads the guest control registers (CR3, CR4) into the guest-state area
3875 * in the VMCS.
3876 *
3877 * @returns VBox strict status code.
3878 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3879 * without unrestricted guest access and the VMMDev is not presently
3880 * mapped (e.g. EFI32).
3881 *
3882 * @param pVCpu The cross context virtual CPU structure.
3883 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3884 * out-of-sync. Make sure to update the required fields
3885 * before using them.
3886 *
3887 * @remarks No-long-jump zone!!!
3888 */
3889static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3890{
3891 int rc = VINF_SUCCESS;
3892 PVM pVM = pVCpu->CTX_SUFF(pVM);
3893
3894 /*
3895 * Guest CR2.
3896 * It's always loaded in the assembler code. Nothing to do here.
3897 */
3898
3899 /*
3900 * Guest CR3.
3901 */
3902 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3903 {
3904 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3905 if (pVM->hm.s.fNestedPaging)
3906 {
3907 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3908
3909 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3910 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3911 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3912 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3913
3914 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3915 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3916 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3917
3918 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3919 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3920 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3921 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3922 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3923 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3924 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3925
3926 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3927 AssertRCReturn(rc, rc);
3928 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3929
3930 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3931 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3932 {
3933 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3934 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3935 {
3936 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3937 AssertRCReturn(rc, rc);
3938 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3939 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3940 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3941 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3942 AssertRCReturn(rc, rc);
3943 }
3944
3945 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3946 have Unrestricted Execution to handle the guest when it's not using paging. */
3947 GCPhysGuestCR3 = pMixedCtx->cr3;
3948 }
3949 else
3950 {
3951 /*
3952 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3953 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3954 * EPT takes care of translating it to host-physical addresses.
3955 */
3956 RTGCPHYS GCPhys;
3957 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3958
3959 /* We obtain it here every time as the guest could have relocated this PCI region. */
3960 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3961 if (RT_SUCCESS(rc))
3962 { /* likely */ }
3963 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3964 {
3965 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3966 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3967 }
3968 else
3969 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3970
3971 GCPhysGuestCR3 = GCPhys;
3972 }
3973
3974 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3975 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3976 }
3977 else
3978 {
3979 /* Non-nested paging case, just use the hypervisor's CR3. */
3980 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3981
3982 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3983 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3984 }
3985 AssertRCReturn(rc, rc);
3986
3987 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3988 }
3989
3990 /*
3991 * Guest CR4.
3992 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3993 */
3994 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3995 {
3996 Assert(!(pMixedCtx->cr4 >> 32));
3997 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3998
3999 /* The guest's view of its CR4 is unblemished. */
4000 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4001 AssertRCReturn(rc, rc);
4002 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4003
4004 /* Setup VT-x's view of the guest CR4. */
4005 /*
4006 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4007 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4008 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4009 */
4010 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4011 {
4012 Assert(pVM->hm.s.vmx.pRealModeTSS);
4013 Assert(PDMVmmDevHeapIsEnabled(pVM));
4014 u32GuestCR4 &= ~X86_CR4_VME;
4015 }
4016
4017 if (pVM->hm.s.fNestedPaging)
4018 {
4019 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4020 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4021 {
4022 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4023 u32GuestCR4 |= X86_CR4_PSE;
4024 /* Our identity mapping is a 32-bit page directory. */
4025 u32GuestCR4 &= ~X86_CR4_PAE;
4026 }
4027 /* else use guest CR4.*/
4028 }
4029 else
4030 {
4031 /*
4032 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4033 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4034 */
4035 switch (pVCpu->hm.s.enmShadowMode)
4036 {
4037 case PGMMODE_REAL: /* Real-mode. */
4038 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4039 case PGMMODE_32_BIT: /* 32-bit paging. */
4040 {
4041 u32GuestCR4 &= ~X86_CR4_PAE;
4042 break;
4043 }
4044
4045 case PGMMODE_PAE: /* PAE paging. */
4046 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4047 {
4048 u32GuestCR4 |= X86_CR4_PAE;
4049 break;
4050 }
4051
4052 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4053 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4054#ifdef VBOX_ENABLE_64_BITS_GUESTS
4055 break;
4056#endif
4057 default:
4058 AssertFailed();
4059 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4060 }
4061 }
4062
4063 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4064 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4065 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4066 u32GuestCR4 |= uSetCR4;
4067 u32GuestCR4 &= uZapCR4;
4068
4069 /* Write VT-x's view of the guest CR4 into the VMCS. */
4070 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4071 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4072 AssertRCReturn(rc, rc);
4073
4074 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4075 uint32_t u32CR4Mask = X86_CR4_VME
4076 | X86_CR4_PAE
4077 | X86_CR4_PGE
4078 | X86_CR4_PSE
4079 | X86_CR4_VMXE;
4080 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4081 u32CR4Mask |= X86_CR4_OSXSAVE;
4082 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4083 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4084 AssertRCReturn(rc, rc);
4085
4086 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4087 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4088
4089 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4090 }
4091 return rc;
4092}
4093
4094
4095/**
4096 * Loads the guest debug registers into the guest-state area in the VMCS.
4097 *
4098 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4099 *
4100 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4101 *
4102 * @returns VBox status code.
4103 * @param pVCpu The cross context virtual CPU structure.
4104 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4105 * out-of-sync. Make sure to update the required fields
4106 * before using them.
4107 *
4108 * @remarks No-long-jump zone!!!
4109 */
4110static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4111{
4112 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4113 return VINF_SUCCESS;
4114
4115#ifdef VBOX_STRICT
4116 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4117 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4118 {
4119 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4120 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4121 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4122 }
4123#endif
4124
4125 int rc;
4126 PVM pVM = pVCpu->CTX_SUFF(pVM);
4127 bool fSteppingDB = false;
4128 bool fInterceptMovDRx = false;
4129 if (pVCpu->hm.s.fSingleInstruction)
4130 {
4131 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4132 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4133 {
4134 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4136 AssertRCReturn(rc, rc);
4137 Assert(fSteppingDB == false);
4138 }
4139 else
4140 {
4141 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4142 pVCpu->hm.s.fClearTrapFlag = true;
4143 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4144 fSteppingDB = true;
4145 }
4146 }
4147
4148 if ( fSteppingDB
4149 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4150 {
4151 /*
4152 * Use the combined guest and host DRx values found in the hypervisor
4153 * register set because the debugger has breakpoints active or someone
4154 * is single stepping on the host side without a monitor trap flag.
4155 *
4156 * Note! DBGF expects a clean DR6 state before executing guest code.
4157 */
4158#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4159 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4160 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4161 {
4162 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4163 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4164 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4165 }
4166 else
4167#endif
4168 if (!CPUMIsHyperDebugStateActive(pVCpu))
4169 {
4170 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4171 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4172 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4173 }
4174
4175 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4176 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4177 AssertRCReturn(rc, rc);
4178
4179 pVCpu->hm.s.fUsingHyperDR7 = true;
4180 fInterceptMovDRx = true;
4181 }
4182 else
4183 {
4184 /*
4185 * If the guest has enabled debug registers, we need to load them prior to
4186 * executing guest code so they'll trigger at the right time.
4187 */
4188 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4189 {
4190#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4191 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4192 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4193 {
4194 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4195 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4196 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4197 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4198 }
4199 else
4200#endif
4201 if (!CPUMIsGuestDebugStateActive(pVCpu))
4202 {
4203 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4204 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4205 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4206 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4207 }
4208 Assert(!fInterceptMovDRx);
4209 }
4210 /*
4211 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4212 * must intercept #DB in order to maintain a correct DR6 guest value, and
4213 * because we need to intercept it to prevent nested #DBs from hanging the
4214 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4215 */
4216#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4217 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4218 && !CPUMIsGuestDebugStateActive(pVCpu))
4219#else
4220 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4221#endif
4222 {
4223 fInterceptMovDRx = true;
4224 }
4225
4226 /* Update guest DR7. */
4227 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4228 AssertRCReturn(rc, rc);
4229
4230 pVCpu->hm.s.fUsingHyperDR7 = false;
4231 }
4232
4233 /*
4234 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4235 */
4236 if (fInterceptMovDRx)
4237 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4238 else
4239 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4241 AssertRCReturn(rc, rc);
4242
4243 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4244 return VINF_SUCCESS;
4245}
4246
4247
4248#ifdef VBOX_STRICT
4249/**
4250 * Strict function to validate segment registers.
4251 *
4252 * @remarks ASSUMES CR0 is up to date.
4253 */
4254static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4255{
4256 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4257 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4258 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4259 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4260 && ( !CPUMIsGuestInRealModeEx(pCtx)
4261 && !CPUMIsGuestInV86ModeEx(pCtx)))
4262 {
4263 /* Protected mode checks */
4264 /* CS */
4265 Assert(pCtx->cs.Attr.n.u1Present);
4266 Assert(!(pCtx->cs.Attr.u & 0xf00));
4267 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4268 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4269 || !(pCtx->cs.Attr.n.u1Granularity));
4270 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4271 || (pCtx->cs.Attr.n.u1Granularity));
4272 /* CS cannot be loaded with NULL in protected mode. */
4273 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4274 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4275 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4276 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4277 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4278 else
4279 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4280 /* SS */
4281 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4282 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4283 if ( !(pCtx->cr0 & X86_CR0_PE)
4284 || pCtx->cs.Attr.n.u4Type == 3)
4285 {
4286 Assert(!pCtx->ss.Attr.n.u2Dpl);
4287 }
4288 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4289 {
4290 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4291 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4292 Assert(pCtx->ss.Attr.n.u1Present);
4293 Assert(!(pCtx->ss.Attr.u & 0xf00));
4294 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4295 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4296 || !(pCtx->ss.Attr.n.u1Granularity));
4297 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4298 || (pCtx->ss.Attr.n.u1Granularity));
4299 }
4300 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4301 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4302 {
4303 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4304 Assert(pCtx->ds.Attr.n.u1Present);
4305 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4306 Assert(!(pCtx->ds.Attr.u & 0xf00));
4307 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->ds.Attr.n.u1Granularity));
4310 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4311 || (pCtx->ds.Attr.n.u1Granularity));
4312 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4313 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4314 }
4315 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4316 {
4317 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4318 Assert(pCtx->es.Attr.n.u1Present);
4319 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4320 Assert(!(pCtx->es.Attr.u & 0xf00));
4321 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4322 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4323 || !(pCtx->es.Attr.n.u1Granularity));
4324 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4325 || (pCtx->es.Attr.n.u1Granularity));
4326 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4327 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4328 }
4329 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4330 {
4331 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4332 Assert(pCtx->fs.Attr.n.u1Present);
4333 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4334 Assert(!(pCtx->fs.Attr.u & 0xf00));
4335 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4336 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4337 || !(pCtx->fs.Attr.n.u1Granularity));
4338 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4339 || (pCtx->fs.Attr.n.u1Granularity));
4340 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4341 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4342 }
4343 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4344 {
4345 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4346 Assert(pCtx->gs.Attr.n.u1Present);
4347 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4348 Assert(!(pCtx->gs.Attr.u & 0xf00));
4349 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4350 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4351 || !(pCtx->gs.Attr.n.u1Granularity));
4352 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4353 || (pCtx->gs.Attr.n.u1Granularity));
4354 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4355 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4356 }
4357 /* 64-bit capable CPUs. */
4358# if HC_ARCH_BITS == 64
4359 Assert(!(pCtx->cs.u64Base >> 32));
4360 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4361 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4362 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4363# endif
4364 }
4365 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4366 || ( CPUMIsGuestInRealModeEx(pCtx)
4367 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4368 {
4369 /* Real and v86 mode checks. */
4370 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4371 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4372 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4373 {
4374 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4375 }
4376 else
4377 {
4378 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4379 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4380 }
4381
4382 /* CS */
4383 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4384 Assert(pCtx->cs.u32Limit == 0xffff);
4385 Assert(u32CSAttr == 0xf3);
4386 /* SS */
4387 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4388 Assert(pCtx->ss.u32Limit == 0xffff);
4389 Assert(u32SSAttr == 0xf3);
4390 /* DS */
4391 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4392 Assert(pCtx->ds.u32Limit == 0xffff);
4393 Assert(u32DSAttr == 0xf3);
4394 /* ES */
4395 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4396 Assert(pCtx->es.u32Limit == 0xffff);
4397 Assert(u32ESAttr == 0xf3);
4398 /* FS */
4399 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4400 Assert(pCtx->fs.u32Limit == 0xffff);
4401 Assert(u32FSAttr == 0xf3);
4402 /* GS */
4403 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4404 Assert(pCtx->gs.u32Limit == 0xffff);
4405 Assert(u32GSAttr == 0xf3);
4406 /* 64-bit capable CPUs. */
4407# if HC_ARCH_BITS == 64
4408 Assert(!(pCtx->cs.u64Base >> 32));
4409 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4410 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4411 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu The cross context virtual CPU structure.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4436 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4437 AssertRCReturn(rc, rc);
4438
4439 uint32_t u32Access = pSelReg->Attr.u;
4440 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4441 {
4442 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4443 u32Access = 0xf3;
4444 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4445 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4446 }
4447 else
4448 {
4449 /*
4450 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4451 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4452 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4453 * loaded in protected-mode have their attribute as 0.
4454 */
4455 if (!u32Access)
4456 u32Access = X86DESCATTR_UNUSABLE;
4457 }
4458
4459 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4460 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4461 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4462
4463 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4464 AssertRCReturn(rc, rc);
4465 return rc;
4466}
4467
4468
4469/**
4470 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4471 * into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVCpu The cross context virtual CPU structure.
4475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4476 * out-of-sync. Make sure to update the required fields
4477 * before using them.
4478 *
4479 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4480 * @remarks No-long-jump zone!!!
4481 */
4482static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4483{
4484 int rc = VERR_INTERNAL_ERROR_5;
4485 PVM pVM = pVCpu->CTX_SUFF(pVM);
4486
4487 /*
4488 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4489 */
4490 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4491 {
4492 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4493 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4494 {
4495 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4496 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4501 }
4502
4503#ifdef VBOX_WITH_REM
4504 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4505 {
4506 Assert(pVM->hm.s.vmx.pRealModeTSS);
4507 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4508 if ( pVCpu->hm.s.vmx.fWasInRealMode
4509 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4510 {
4511 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4512 in real-mode (e.g. OpenBSD 4.0) */
4513 REMFlushTBs(pVM);
4514 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4515 pVCpu->hm.s.vmx.fWasInRealMode = false;
4516 }
4517 }
4518#endif
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4520 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4523 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4526 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4529 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4532 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4535 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4536 AssertRCReturn(rc, rc);
4537
4538#ifdef VBOX_STRICT
4539 /* Validate. */
4540 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4541#endif
4542
4543 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4544 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4545 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4546 }
4547
4548 /*
4549 * Guest TR.
4550 */
4551 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4552 {
4553 /*
4554 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4555 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4556 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4557 */
4558 uint16_t u16Sel = 0;
4559 uint32_t u32Limit = 0;
4560 uint64_t u64Base = 0;
4561 uint32_t u32AccessRights = 0;
4562
4563 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4564 {
4565 u16Sel = pMixedCtx->tr.Sel;
4566 u32Limit = pMixedCtx->tr.u32Limit;
4567 u64Base = pMixedCtx->tr.u64Base;
4568 u32AccessRights = pMixedCtx->tr.Attr.u;
4569 }
4570 else
4571 {
4572 Assert(pVM->hm.s.vmx.pRealModeTSS);
4573 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4574
4575 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4576 RTGCPHYS GCPhys;
4577 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4578 AssertRCReturn(rc, rc);
4579
4580 X86DESCATTR DescAttr;
4581 DescAttr.u = 0;
4582 DescAttr.n.u1Present = 1;
4583 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4584
4585 u16Sel = 0;
4586 u32Limit = HM_VTX_TSS_SIZE;
4587 u64Base = GCPhys; /* in real-mode phys = virt. */
4588 u32AccessRights = DescAttr.u;
4589 }
4590
4591 /* Validate. */
4592 Assert(!(u16Sel & RT_BIT(2)));
4593 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4594 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4595 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4596 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4597 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4598 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4599 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4600 Assert( (u32Limit & 0xfff) == 0xfff
4601 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4602 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4603 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4604
4605 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4606 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4607 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4609 AssertRCReturn(rc, rc);
4610
4611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4613 }
4614
4615 /*
4616 * Guest GDTR.
4617 */
4618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4621 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4622 AssertRCReturn(rc, rc);
4623
4624 /* Validate. */
4625 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4626
4627 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4628 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4629 }
4630
4631 /*
4632 * Guest LDTR.
4633 */
4634 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4635 {
4636 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4637 uint32_t u32Access = 0;
4638 if (!pMixedCtx->ldtr.Attr.u)
4639 u32Access = X86DESCATTR_UNUSABLE;
4640 else
4641 u32Access = pMixedCtx->ldtr.Attr.u;
4642
4643 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4647 AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4674 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4675 AssertRCReturn(rc, rc);
4676
4677 /* Validate. */
4678 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4679
4680 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4681 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4682 }
4683
4684 return VINF_SUCCESS;
4685}
4686
4687
4688/**
4689 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4690 * areas.
4691 *
4692 * These MSRs will automatically be loaded to the host CPU on every successful
4693 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4694 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4695 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4696 *
4697 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4698 *
4699 * @returns VBox status code.
4700 * @param pVCpu The cross context virtual CPU structure.
4701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4702 * out-of-sync. Make sure to update the required fields
4703 * before using them.
4704 *
4705 * @remarks No-long-jump zone!!!
4706 */
4707static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4708{
4709 AssertPtr(pVCpu);
4710 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4711
4712 /*
4713 * MSRs that we use the auto-load/store MSR area in the VMCS.
4714 */
4715 PVM pVM = pVCpu->CTX_SUFF(pVM);
4716 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4717 {
4718 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4719#if HC_ARCH_BITS == 32
4720 if (pVM->hm.s.fAllow64BitGuests)
4721 {
4722 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4723 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4724 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4726 AssertRCReturn(rc, rc);
4727# ifdef LOG_ENABLED
4728 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4729 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4730 {
4731 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4732 pMsr->u64Value));
4733 }
4734# endif
4735 }
4736#endif
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738 }
4739
4740 /*
4741 * Guest Sysenter MSRs.
4742 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743 * VM-exits on WRMSRs for these MSRs.
4744 */
4745 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4746 {
4747 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4748 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4749 }
4750
4751 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4752 {
4753 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4754 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4755 }
4756
4757 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4758 {
4759 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4760 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4761 }
4762
4763 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4764 {
4765 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4766 {
4767 /*
4768 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4769 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4770 */
4771 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4772 {
4773 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4774 AssertRCReturn(rc,rc);
4775 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4776 }
4777 else
4778 {
4779 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4780 NULL /* pfAddedAndUpdated */);
4781 AssertRCReturn(rc, rc);
4782
4783 /* We need to intercept reads too, see @bugref{7386#c16}. */
4784 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4785 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4786 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4787 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4788 }
4789 }
4790 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4791 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4792 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4793 }
4794
4795 return VINF_SUCCESS;
4796}
4797
4798
4799/**
4800 * Loads the guest activity state into the guest-state area in the VMCS.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu The cross context virtual CPU structure.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 *
4808 * @remarks No-long-jump zone!!!
4809 */
4810static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811{
4812 NOREF(pMixedCtx);
4813 /** @todo See if we can make use of other states, e.g.
4814 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4815 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4816 {
4817 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4818 AssertRCReturn(rc, rc);
4819
4820 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4821 }
4822 return VINF_SUCCESS;
4823}
4824
4825
4826#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4827/**
4828 * Check if guest state allows safe use of 32-bit switcher again.
4829 *
4830 * Segment bases and protected mode structures must be 32-bit addressable
4831 * because the 32-bit switcher will ignore high dword when writing these VMCS
4832 * fields. See @bugref{8432} for details.
4833 *
4834 * @returns true if safe, false if must continue to use the 64-bit switcher.
4835 * @param pVCpu The cross context virtual CPU structure.
4836 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4837 * out-of-sync. Make sure to update the required fields
4838 * before using them.
4839 *
4840 * @remarks No-long-jump zone!!!
4841 */
4842static bool hmR0VmxIs32BitSwitcherSafe(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4843{
4844 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4845 return false;
4846 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4847 return false;
4848 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4849 return false;
4850 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4851 return false;
4852 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4853 return false;
4854 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4855 return false;
4856 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4857 return false;
4858 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4859 return false;
4860 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4861 return false;
4862 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4863 return false;
4864 /* All good, bases are 32-bit. */
4865 return true;
4866}
4867#endif
4868
4869
4870/**
4871 * Sets up the appropriate function to run guest code.
4872 *
4873 * @returns VBox status code.
4874 * @param pVCpu The cross context virtual CPU structure.
4875 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4876 * out-of-sync. Make sure to update the required fields
4877 * before using them.
4878 *
4879 * @remarks No-long-jump zone!!!
4880 */
4881static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4882{
4883 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4884 {
4885#ifndef VBOX_ENABLE_64_BITS_GUESTS
4886 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4887#endif
4888 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4889#if HC_ARCH_BITS == 32
4890 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4891 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4892 {
4893 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4894 {
4895 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4896 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4897 | HM_CHANGED_VMX_ENTRY_CTLS
4898 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4899 }
4900 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4901
4902 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4903 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4904 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4905 }
4906#else
4907 /* 64-bit host. */
4908 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4909#endif
4910 }
4911 else
4912 {
4913 /* Guest is not in long mode, use the 32-bit handler. */
4914#if HC_ARCH_BITS == 32
4915 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4916 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4917 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4918 {
4919 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4920 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4921 | HM_CHANGED_VMX_ENTRY_CTLS
4922 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4923 }
4924# ifdef VBOX_ENABLE_64_BITS_GUESTS
4925 /* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design. See @bugref{8432#c7}.
4926 * Except if Real-on-V86 is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4927 * state where it's safe to use the 32-bit switcher again.
4928 */
4929 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4930 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4931
4932 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4933 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4934 else
4935 {
4936 Assert(!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active);
4937 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4938 if (hmR0VmxIs32BitSwitcherSafe(pVCpu, pMixedCtx))
4939 {
4940 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4941 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4942 }
4943 }
4944# else
4945 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4946# endif
4947#else
4948 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4949#endif
4950 }
4951 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4952 return VINF_SUCCESS;
4953}
4954
4955
4956/**
4957 * Wrapper for running the guest code in VT-x.
4958 *
4959 * @returns VBox status code, no informational status codes.
4960 * @param pVM The cross context VM structure.
4961 * @param pVCpu The cross context virtual CPU structure.
4962 * @param pCtx Pointer to the guest-CPU context.
4963 *
4964 * @remarks No-long-jump zone!!!
4965 */
4966DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4967{
4968 /*
4969 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4970 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4971 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4972 */
4973 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4974 /** @todo Add stats for resume vs launch. */
4975#ifdef VBOX_WITH_KERNEL_USING_XMM
4976 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4977#else
4978 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4979#endif
4980 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4981 return rc;
4982}
4983
4984
4985/**
4986 * Reports world-switch error and dumps some useful debug info.
4987 *
4988 * @param pVM The cross context VM structure.
4989 * @param pVCpu The cross context virtual CPU structure.
4990 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4991 * @param pCtx Pointer to the guest-CPU context.
4992 * @param pVmxTransient Pointer to the VMX transient structure (only
4993 * exitReason updated).
4994 */
4995static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4996{
4997 Assert(pVM);
4998 Assert(pVCpu);
4999 Assert(pCtx);
5000 Assert(pVmxTransient);
5001 HMVMX_ASSERT_PREEMPT_SAFE();
5002
5003 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5004 switch (rcVMRun)
5005 {
5006 case VERR_VMX_INVALID_VMXON_PTR:
5007 AssertFailed();
5008 break;
5009 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5010 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5011 {
5012 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5013 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5014 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5015 AssertRC(rc);
5016
5017 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5018 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5019 Cannot do it here as we may have been long preempted. */
5020
5021#ifdef VBOX_STRICT
5022 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5023 pVmxTransient->uExitReason));
5024 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5025 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5026 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5027 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5028 else
5029 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5030 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5031 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5032
5033 /* VMX control bits. */
5034 uint32_t u32Val;
5035 uint64_t u64Val;
5036 RTHCUINTREG uHCReg;
5037 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5038 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5039 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5040 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5041 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5042 {
5043 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5044 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5045 }
5046 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5050 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5051 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5054 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5055 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5056 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5057 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5058 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5059 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5060 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5061 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5062 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5063 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5064 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5065 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5066 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5067 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5068 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5069 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5070 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5071 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5072 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5073 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5074 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5075 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5076 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5077 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5078 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5079 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5080 if (pVM->hm.s.fNestedPaging)
5081 {
5082 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5083 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5084 }
5085
5086 /* Guest bits. */
5087 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5088 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5089 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5090 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5091 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5092 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5093 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5094 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5095
5096 /* Host bits. */
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5098 Log4(("Host CR0 %#RHr\n", uHCReg));
5099 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5100 Log4(("Host CR3 %#RHr\n", uHCReg));
5101 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5102 Log4(("Host CR4 %#RHr\n", uHCReg));
5103
5104 RTGDTR HostGdtr;
5105 PCX86DESCHC pDesc;
5106 ASMGetGDTR(&HostGdtr);
5107 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5108 Log4(("Host CS %#08x\n", u32Val));
5109 if (u32Val < HostGdtr.cbGdt)
5110 {
5111 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5112 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5113 }
5114
5115 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5116 Log4(("Host DS %#08x\n", u32Val));
5117 if (u32Val < HostGdtr.cbGdt)
5118 {
5119 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5120 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5121 }
5122
5123 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5124 Log4(("Host ES %#08x\n", u32Val));
5125 if (u32Val < HostGdtr.cbGdt)
5126 {
5127 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5128 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5129 }
5130
5131 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5132 Log4(("Host FS %#08x\n", u32Val));
5133 if (u32Val < HostGdtr.cbGdt)
5134 {
5135 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5136 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5137 }
5138
5139 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5140 Log4(("Host GS %#08x\n", u32Val));
5141 if (u32Val < HostGdtr.cbGdt)
5142 {
5143 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5144 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5145 }
5146
5147 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5148 Log4(("Host SS %#08x\n", u32Val));
5149 if (u32Val < HostGdtr.cbGdt)
5150 {
5151 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5152 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5153 }
5154
5155 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5156 Log4(("Host TR %#08x\n", u32Val));
5157 if (u32Val < HostGdtr.cbGdt)
5158 {
5159 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5160 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5161 }
5162
5163 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5164 Log4(("Host TR Base %#RHv\n", uHCReg));
5165 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5166 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5167 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5168 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5169 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5170 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5171 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5172 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5173 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5174 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5175 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5176 Log4(("Host RSP %#RHv\n", uHCReg));
5177 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5178 Log4(("Host RIP %#RHv\n", uHCReg));
5179# if HC_ARCH_BITS == 64
5180 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5181 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5182 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5183 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5184 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5185 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5186# endif
5187#endif /* VBOX_STRICT */
5188 break;
5189 }
5190
5191 default:
5192 /* Impossible */
5193 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5194 break;
5195 }
5196 NOREF(pVM); NOREF(pCtx);
5197}
5198
5199
5200#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5201#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5202# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5203#endif
5204#ifdef VBOX_STRICT
5205static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5206{
5207 switch (idxField)
5208 {
5209 case VMX_VMCS_GUEST_RIP:
5210 case VMX_VMCS_GUEST_RSP:
5211 case VMX_VMCS_GUEST_SYSENTER_EIP:
5212 case VMX_VMCS_GUEST_SYSENTER_ESP:
5213 case VMX_VMCS_GUEST_GDTR_BASE:
5214 case VMX_VMCS_GUEST_IDTR_BASE:
5215 case VMX_VMCS_GUEST_CS_BASE:
5216 case VMX_VMCS_GUEST_DS_BASE:
5217 case VMX_VMCS_GUEST_ES_BASE:
5218 case VMX_VMCS_GUEST_FS_BASE:
5219 case VMX_VMCS_GUEST_GS_BASE:
5220 case VMX_VMCS_GUEST_SS_BASE:
5221 case VMX_VMCS_GUEST_LDTR_BASE:
5222 case VMX_VMCS_GUEST_TR_BASE:
5223 case VMX_VMCS_GUEST_CR3:
5224 return true;
5225 }
5226 return false;
5227}
5228
5229static bool hmR0VmxIsValidReadField(uint32_t idxField)
5230{
5231 switch (idxField)
5232 {
5233 /* Read-only fields. */
5234 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5235 return true;
5236 }
5237 /* Remaining readable fields should also be writable. */
5238 return hmR0VmxIsValidWriteField(idxField);
5239}
5240#endif /* VBOX_STRICT */
5241
5242
5243/**
5244 * Executes the specified handler in 64-bit mode.
5245 *
5246 * @returns VBox status code (no informational status codes).
5247 * @param pVM The cross context VM structure.
5248 * @param pVCpu The cross context virtual CPU structure.
5249 * @param pCtx Pointer to the guest CPU context.
5250 * @param enmOp The operation to perform.
5251 * @param cParams Number of parameters.
5252 * @param paParam Array of 32-bit parameters.
5253 */
5254VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5255 uint32_t cParams, uint32_t *paParam)
5256{
5257 NOREF(pCtx);
5258
5259 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5260 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5261 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5262 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5263
5264#ifdef VBOX_STRICT
5265 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5266 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5267
5268 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5269 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5270#endif
5271
5272 /* Disable interrupts. */
5273 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5274
5275#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5276 RTCPUID idHostCpu = RTMpCpuId();
5277 CPUMR0SetLApic(pVCpu, idHostCpu);
5278#endif
5279
5280 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5281 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5282
5283 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5284 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5285
5286 /* Leave VMX Root Mode. */
5287 VMXDisable();
5288
5289 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5290
5291 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5292 CPUMSetHyperEIP(pVCpu, enmOp);
5293 for (int i = (int)cParams - 1; i >= 0; i--)
5294 CPUMPushHyper(pVCpu, paParam[i]);
5295
5296 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5297
5298 /* Call the switcher. */
5299 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5300 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5301
5302 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5303 /* Make sure the VMX instructions don't cause #UD faults. */
5304 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5305
5306 /* Re-enter VMX Root Mode */
5307 int rc2 = VMXEnable(HCPhysCpuPage);
5308 if (RT_FAILURE(rc2))
5309 {
5310 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5311 ASMSetFlags(fOldEFlags);
5312 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5313 return rc2;
5314 }
5315
5316 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5317 AssertRC(rc2);
5318 Assert(!(ASMGetFlags() & X86_EFL_IF));
5319 ASMSetFlags(fOldEFlags);
5320 return rc;
5321}
5322
5323
5324/**
5325 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5326 * supporting 64-bit guests.
5327 *
5328 * @returns VBox status code.
5329 * @param fResume Whether to VMLAUNCH or VMRESUME.
5330 * @param pCtx Pointer to the guest-CPU context.
5331 * @param pCache Pointer to the VMCS cache.
5332 * @param pVM The cross context VM structure.
5333 * @param pVCpu The cross context virtual CPU structure.
5334 */
5335DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5336{
5337 NOREF(fResume);
5338
5339 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5340 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5341
5342#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5343 pCache->uPos = 1;
5344 pCache->interPD = PGMGetInterPaeCR3(pVM);
5345 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5346#endif
5347
5348#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5349 pCache->TestIn.HCPhysCpuPage = 0;
5350 pCache->TestIn.HCPhysVmcs = 0;
5351 pCache->TestIn.pCache = 0;
5352 pCache->TestOut.HCPhysVmcs = 0;
5353 pCache->TestOut.pCache = 0;
5354 pCache->TestOut.pCtx = 0;
5355 pCache->TestOut.eflags = 0;
5356#else
5357 NOREF(pCache);
5358#endif
5359
5360 uint32_t aParam[10];
5361 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5362 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5363 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5364 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5365 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5366 aParam[5] = 0;
5367 aParam[6] = VM_RC_ADDR(pVM, pVM);
5368 aParam[7] = 0;
5369 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5370 aParam[9] = 0;
5371
5372#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5373 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5374 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5375#endif
5376 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5377
5378#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5379 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5380 Assert(pCtx->dr[4] == 10);
5381 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5382#endif
5383
5384#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5385 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5386 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5387 pVCpu->hm.s.vmx.HCPhysVmcs));
5388 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5389 pCache->TestOut.HCPhysVmcs));
5390 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5391 pCache->TestOut.pCache));
5392 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5393 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5394 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5395 pCache->TestOut.pCtx));
5396 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5397#endif
5398 return rc;
5399}
5400
5401
5402/**
5403 * Initialize the VMCS-Read cache.
5404 *
5405 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5406 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5407 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5408 * (those that have a 32-bit FULL & HIGH part).
5409 *
5410 * @returns VBox status code.
5411 * @param pVM The cross context VM structure.
5412 * @param pVCpu The cross context virtual CPU structure.
5413 */
5414static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5415{
5416#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5417{ \
5418 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5419 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5420 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5421 ++cReadFields; \
5422}
5423
5424 AssertPtr(pVM);
5425 AssertPtr(pVCpu);
5426 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5427 uint32_t cReadFields = 0;
5428
5429 /*
5430 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5431 * and serve to indicate exceptions to the rules.
5432 */
5433
5434 /* Guest-natural selector base fields. */
5435#if 0
5436 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5438 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5439#endif
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5443 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5444 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5445 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5446 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5447 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5452#if 0
5453 /* Unused natural width guest-state fields. */
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5456#endif
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5459
5460 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5461#if 0
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5471#endif
5472
5473 /* Natural width guest-state fields. */
5474 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5475#if 0
5476 /* Currently unused field. */
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5478#endif
5479
5480 if (pVM->hm.s.fNestedPaging)
5481 {
5482 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5483 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5484 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5485 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5486 }
5487 else
5488 {
5489 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5490 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5491 }
5492
5493#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5494 return VINF_SUCCESS;
5495}
5496
5497
5498/**
5499 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5500 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5501 * darwin, running 64-bit guests).
5502 *
5503 * @returns VBox status code.
5504 * @param pVCpu The cross context virtual CPU structure.
5505 * @param idxField The VMCS field encoding.
5506 * @param u64Val 16, 32 or 64-bit value.
5507 */
5508VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5509{
5510 int rc;
5511 switch (idxField)
5512 {
5513 /*
5514 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5515 */
5516 /* 64-bit Control fields. */
5517 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5518 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5519 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5520 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5521 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5522 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5523 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5524 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5525 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5526 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5527 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5528 case VMX_VMCS64_CTRL_EPTP_FULL:
5529 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5530 /* 64-bit Guest-state fields. */
5531 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5532 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5533 case VMX_VMCS64_GUEST_PAT_FULL:
5534 case VMX_VMCS64_GUEST_EFER_FULL:
5535 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5536 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5537 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5538 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5539 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5540 /* 64-bit Host-state fields. */
5541 case VMX_VMCS64_HOST_PAT_FULL:
5542 case VMX_VMCS64_HOST_EFER_FULL:
5543 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5544 {
5545 rc = VMXWriteVmcs32(idxField, u64Val);
5546 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5547 break;
5548 }
5549
5550 /*
5551 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5552 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5553 */
5554 /* Natural-width Guest-state fields. */
5555 case VMX_VMCS_GUEST_CR3:
5556 case VMX_VMCS_GUEST_ES_BASE:
5557 case VMX_VMCS_GUEST_CS_BASE:
5558 case VMX_VMCS_GUEST_SS_BASE:
5559 case VMX_VMCS_GUEST_DS_BASE:
5560 case VMX_VMCS_GUEST_FS_BASE:
5561 case VMX_VMCS_GUEST_GS_BASE:
5562 case VMX_VMCS_GUEST_LDTR_BASE:
5563 case VMX_VMCS_GUEST_TR_BASE:
5564 case VMX_VMCS_GUEST_GDTR_BASE:
5565 case VMX_VMCS_GUEST_IDTR_BASE:
5566 case VMX_VMCS_GUEST_RSP:
5567 case VMX_VMCS_GUEST_RIP:
5568 case VMX_VMCS_GUEST_SYSENTER_ESP:
5569 case VMX_VMCS_GUEST_SYSENTER_EIP:
5570 {
5571 if (!(u64Val >> 32))
5572 {
5573 /* If this field is 64-bit, VT-x will zero out the top bits. */
5574 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5575 }
5576 else
5577 {
5578 /* Assert that only the 32->64 switcher case should ever come here. */
5579 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5580 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5581 }
5582 break;
5583 }
5584
5585 default:
5586 {
5587 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5588 rc = VERR_INVALID_PARAMETER;
5589 break;
5590 }
5591 }
5592 AssertRCReturn(rc, rc);
5593 return rc;
5594}
5595
5596
5597/**
5598 * Queue up a VMWRITE by using the VMCS write cache.
5599 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5600 *
5601 * @param pVCpu The cross context virtual CPU structure.
5602 * @param idxField The VMCS field encoding.
5603 * @param u64Val 16, 32 or 64-bit value.
5604 */
5605VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5606{
5607 AssertPtr(pVCpu);
5608 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5609
5610 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5611 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5612
5613 /* Make sure there are no duplicates. */
5614 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5615 {
5616 if (pCache->Write.aField[i] == idxField)
5617 {
5618 pCache->Write.aFieldVal[i] = u64Val;
5619 return VINF_SUCCESS;
5620 }
5621 }
5622
5623 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5624 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5625 pCache->Write.cValidEntries++;
5626 return VINF_SUCCESS;
5627}
5628#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5629
5630
5631/**
5632 * Sets up the usage of TSC-offsetting and updates the VMCS.
5633 *
5634 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5635 * VMX preemption timer.
5636 *
5637 * @returns VBox status code.
5638 * @param pVM The cross context VM structure.
5639 * @param pVCpu The cross context virtual CPU structure.
5640 *
5641 * @remarks No-long-jump zone!!!
5642 */
5643static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5644{
5645 int rc;
5646 bool fOffsettedTsc;
5647 bool fParavirtTsc;
5648 if (pVM->hm.s.vmx.fUsePreemptTimer)
5649 {
5650 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5651 &fOffsettedTsc, &fParavirtTsc);
5652
5653 /* Make sure the returned values have sane upper and lower boundaries. */
5654 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5655 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5656 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5657 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5658
5659 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5660 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5661 }
5662 else
5663 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5664
5665 /** @todo later optimize this to be done elsewhere and not before every
5666 * VM-entry. */
5667 if (fParavirtTsc)
5668 {
5669 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5670 information before every VM-entry, hence disable it for performance sake. */
5671#if 0
5672 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5673 AssertRC(rc);
5674#endif
5675 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5676 }
5677
5678 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5679 {
5680 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5681 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5682
5683 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5684 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5685 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5686 }
5687 else
5688 {
5689 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5690 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5691 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5692 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5693 }
5694}
5695
5696
5697/**
5698 * Determines if an exception is a contributory exception.
5699 *
5700 * Contributory exceptions are ones which can cause double-faults unless the
5701 * original exception was a benign exception. Page-fault is intentionally not
5702 * included here as it's a conditional contributory exception.
5703 *
5704 * @returns true if the exception is contributory, false otherwise.
5705 * @param uVector The exception vector.
5706 */
5707DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5708{
5709 switch (uVector)
5710 {
5711 case X86_XCPT_GP:
5712 case X86_XCPT_SS:
5713 case X86_XCPT_NP:
5714 case X86_XCPT_TS:
5715 case X86_XCPT_DE:
5716 return true;
5717 default:
5718 break;
5719 }
5720 return false;
5721}
5722
5723
5724/**
5725 * Sets an event as a pending event to be injected into the guest.
5726 *
5727 * @param pVCpu The cross context virtual CPU structure.
5728 * @param u32IntInfo The VM-entry interruption-information field.
5729 * @param cbInstr The VM-entry instruction length in bytes (for software
5730 * interrupts, exceptions and privileged software
5731 * exceptions).
5732 * @param u32ErrCode The VM-entry exception error code.
5733 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5734 * page-fault.
5735 *
5736 * @remarks Statistics counter assumes this is a guest event being injected or
5737 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5738 * always incremented.
5739 */
5740DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5741 RTGCUINTPTR GCPtrFaultAddress)
5742{
5743 Assert(!pVCpu->hm.s.Event.fPending);
5744 pVCpu->hm.s.Event.fPending = true;
5745 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5746 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5747 pVCpu->hm.s.Event.cbInstr = cbInstr;
5748 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5749}
5750
5751
5752/**
5753 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5754 *
5755 * @param pVCpu The cross context virtual CPU structure.
5756 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5757 * out-of-sync. Make sure to update the required fields
5758 * before using them.
5759 */
5760DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5761{
5762 NOREF(pMixedCtx);
5763 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5764 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5765 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5766 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5767}
5768
5769
5770/**
5771 * Handle a condition that occurred while delivering an event through the guest
5772 * IDT.
5773 *
5774 * @returns Strict VBox status code (i.e. informational status codes too).
5775 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5776 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5777 * to continue execution of the guest which will delivery the \#DF.
5778 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5779 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5780 *
5781 * @param pVCpu The cross context virtual CPU structure.
5782 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5783 * out-of-sync. Make sure to update the required fields
5784 * before using them.
5785 * @param pVmxTransient Pointer to the VMX transient structure.
5786 *
5787 * @remarks No-long-jump zone!!!
5788 */
5789static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5790{
5791 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5792
5793 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5794 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5795
5796 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5797 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5798 {
5799 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5800 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5801
5802 typedef enum
5803 {
5804 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5805 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5806 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5807 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5808 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5809 } VMXREFLECTXCPT;
5810
5811 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5812 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5813 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5814 {
5815 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5816 {
5817 enmReflect = VMXREFLECTXCPT_XCPT;
5818#ifdef VBOX_STRICT
5819 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5820 && uExitVector == X86_XCPT_PF)
5821 {
5822 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5823 }
5824#endif
5825 if ( uExitVector == X86_XCPT_PF
5826 && uIdtVector == X86_XCPT_PF)
5827 {
5828 pVmxTransient->fVectoringDoublePF = true;
5829 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5830 }
5831 else if ( uExitVector == X86_XCPT_AC
5832 && uIdtVector == X86_XCPT_AC)
5833 {
5834 enmReflect = VMXREFLECTXCPT_HANG;
5835 Log4(("IDT: Nested #AC - Bad guest\n"));
5836 }
5837 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5838 && hmR0VmxIsContributoryXcpt(uExitVector)
5839 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5840 || uIdtVector == X86_XCPT_PF))
5841 {
5842 enmReflect = VMXREFLECTXCPT_DF;
5843 }
5844 else if (uIdtVector == X86_XCPT_DF)
5845 enmReflect = VMXREFLECTXCPT_TF;
5846 }
5847 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5848 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5849 {
5850 /*
5851 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5852 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5853 */
5854 enmReflect = VMXREFLECTXCPT_XCPT;
5855
5856 if (uExitVector == X86_XCPT_PF)
5857 {
5858 pVmxTransient->fVectoringPF = true;
5859 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5860 }
5861 }
5862 }
5863 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5864 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5865 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5866 {
5867 /*
5868 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5869 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5870 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5871 */
5872 enmReflect = VMXREFLECTXCPT_XCPT;
5873 }
5874
5875 /*
5876 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5877 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5878 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5879 *
5880 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5881 */
5882 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5883 && enmReflect == VMXREFLECTXCPT_XCPT
5884 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5885 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5886 {
5887 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5888 }
5889
5890 switch (enmReflect)
5891 {
5892 case VMXREFLECTXCPT_XCPT:
5893 {
5894 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5895 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5896 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5897
5898 uint32_t u32ErrCode = 0;
5899 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5900 {
5901 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5902 AssertRCReturn(rc2, rc2);
5903 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5904 }
5905
5906 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5907 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5908 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5909 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5910 rcStrict = VINF_SUCCESS;
5911 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5912 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5913
5914 break;
5915 }
5916
5917 case VMXREFLECTXCPT_DF:
5918 {
5919 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5920 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5921 rcStrict = VINF_HM_DOUBLE_FAULT;
5922 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5923 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5924
5925 break;
5926 }
5927
5928 case VMXREFLECTXCPT_TF:
5929 {
5930 rcStrict = VINF_EM_RESET;
5931 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5932 uExitVector));
5933 break;
5934 }
5935
5936 case VMXREFLECTXCPT_HANG:
5937 {
5938 rcStrict = VERR_EM_GUEST_CPU_HANG;
5939 break;
5940 }
5941
5942 default:
5943 Assert(rcStrict == VINF_SUCCESS);
5944 break;
5945 }
5946 }
5947 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5948 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5949 && uExitVector != X86_XCPT_DF
5950 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5951 {
5952 /*
5953 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5954 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5955 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5956 */
5957 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5958 {
5959 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5960 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5961 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5962 }
5963 }
5964
5965 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5966 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5967 return rcStrict;
5968}
5969
5970
5971/**
5972 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5973 *
5974 * @returns VBox status code.
5975 * @param pVCpu The cross context virtual CPU structure.
5976 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5977 * out-of-sync. Make sure to update the required fields
5978 * before using them.
5979 *
5980 * @remarks No-long-jump zone!!!
5981 */
5982static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5983{
5984 NOREF(pMixedCtx);
5985
5986 /*
5987 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5988 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5989 */
5990 VMMRZCallRing3Disable(pVCpu);
5991 HM_DISABLE_PREEMPT();
5992
5993 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5994 {
5995 uint32_t uVal = 0;
5996 uint32_t uShadow = 0;
5997 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5998 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5999 AssertRCReturn(rc, rc);
6000
6001 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6002 CPUMSetGuestCR0(pVCpu, uVal);
6003 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6004 }
6005
6006 HM_RESTORE_PREEMPT();
6007 VMMRZCallRing3Enable(pVCpu);
6008 return VINF_SUCCESS;
6009}
6010
6011
6012/**
6013 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6014 *
6015 * @returns VBox status code.
6016 * @param pVCpu The cross context virtual CPU structure.
6017 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6018 * out-of-sync. Make sure to update the required fields
6019 * before using them.
6020 *
6021 * @remarks No-long-jump zone!!!
6022 */
6023static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6024{
6025 NOREF(pMixedCtx);
6026
6027 int rc = VINF_SUCCESS;
6028 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6029 {
6030 uint32_t uVal = 0;
6031 uint32_t uShadow = 0;
6032 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6033 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6034 AssertRCReturn(rc, rc);
6035
6036 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6037 CPUMSetGuestCR4(pVCpu, uVal);
6038 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6039 }
6040 return rc;
6041}
6042
6043
6044/**
6045 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6046 *
6047 * @returns VBox status code.
6048 * @param pVCpu The cross context virtual CPU structure.
6049 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6050 * out-of-sync. Make sure to update the required fields
6051 * before using them.
6052 *
6053 * @remarks No-long-jump zone!!!
6054 */
6055static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6056{
6057 int rc = VINF_SUCCESS;
6058 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6059 {
6060 uint64_t u64Val = 0;
6061 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6062 AssertRCReturn(rc, rc);
6063
6064 pMixedCtx->rip = u64Val;
6065 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6066 }
6067 return rc;
6068}
6069
6070
6071/**
6072 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6073 *
6074 * @returns VBox status code.
6075 * @param pVCpu The cross context virtual CPU structure.
6076 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6077 * out-of-sync. Make sure to update the required fields
6078 * before using them.
6079 *
6080 * @remarks No-long-jump zone!!!
6081 */
6082static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6083{
6084 int rc = VINF_SUCCESS;
6085 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6086 {
6087 uint64_t u64Val = 0;
6088 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6089 AssertRCReturn(rc, rc);
6090
6091 pMixedCtx->rsp = u64Val;
6092 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6093 }
6094 return rc;
6095}
6096
6097
6098/**
6099 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6100 *
6101 * @returns VBox status code.
6102 * @param pVCpu The cross context virtual CPU structure.
6103 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6104 * out-of-sync. Make sure to update the required fields
6105 * before using them.
6106 *
6107 * @remarks No-long-jump zone!!!
6108 */
6109static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6110{
6111 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6112 {
6113 uint32_t uVal = 0;
6114 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6115 AssertRCReturn(rc, rc);
6116
6117 pMixedCtx->eflags.u32 = uVal;
6118 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6119 {
6120 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6121 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6122
6123 pMixedCtx->eflags.Bits.u1VM = 0;
6124 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6125 }
6126
6127 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6128 }
6129 return VINF_SUCCESS;
6130}
6131
6132
6133/**
6134 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6135 * guest-CPU context.
6136 */
6137DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6138{
6139 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6140 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6141 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6142 return rc;
6143}
6144
6145
6146/**
6147 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6148 * from the guest-state area in the VMCS.
6149 *
6150 * @param pVCpu The cross context virtual CPU structure.
6151 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6152 * out-of-sync. Make sure to update the required fields
6153 * before using them.
6154 *
6155 * @remarks No-long-jump zone!!!
6156 */
6157static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6158{
6159 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6160 {
6161 uint32_t uIntrState = 0;
6162 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6163 AssertRC(rc);
6164
6165 if (!uIntrState)
6166 {
6167 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6168 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6169
6170 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6171 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6172 }
6173 else
6174 {
6175 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6176 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6177 {
6178 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6179 AssertRC(rc);
6180 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6181 AssertRC(rc);
6182
6183 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6184 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6185 }
6186 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6187 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6188
6189 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6190 {
6191 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6192 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6193 }
6194 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6195 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6196 }
6197
6198 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6199 }
6200}
6201
6202
6203/**
6204 * Saves the guest's activity state.
6205 *
6206 * @returns VBox status code.
6207 * @param pVCpu The cross context virtual CPU structure.
6208 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6209 * out-of-sync. Make sure to update the required fields
6210 * before using them.
6211 *
6212 * @remarks No-long-jump zone!!!
6213 */
6214static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6215{
6216 NOREF(pMixedCtx);
6217 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6218 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6219 return VINF_SUCCESS;
6220}
6221
6222
6223/**
6224 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6225 * the current VMCS into the guest-CPU context.
6226 *
6227 * @returns VBox status code.
6228 * @param pVCpu The cross context virtual CPU structure.
6229 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6230 * out-of-sync. Make sure to update the required fields
6231 * before using them.
6232 *
6233 * @remarks No-long-jump zone!!!
6234 */
6235static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6236{
6237 int rc = VINF_SUCCESS;
6238 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6239 {
6240 uint32_t u32Val = 0;
6241 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6242 pMixedCtx->SysEnter.cs = u32Val;
6243 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6244 }
6245
6246 uint64_t u64Val = 0;
6247 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6248 {
6249 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6250 pMixedCtx->SysEnter.eip = u64Val;
6251 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6252 }
6253 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6254 {
6255 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6256 pMixedCtx->SysEnter.esp = u64Val;
6257 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6258 }
6259 return rc;
6260}
6261
6262
6263/**
6264 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6265 * the CPU back into the guest-CPU context.
6266 *
6267 * @returns VBox status code.
6268 * @param pVCpu The cross context virtual CPU structure.
6269 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6270 * out-of-sync. Make sure to update the required fields
6271 * before using them.
6272 *
6273 * @remarks No-long-jump zone!!!
6274 */
6275static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6276{
6277 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6278 VMMRZCallRing3Disable(pVCpu);
6279 HM_DISABLE_PREEMPT();
6280
6281 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6282 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6283 {
6284 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6285 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6286 }
6287
6288 HM_RESTORE_PREEMPT();
6289 VMMRZCallRing3Enable(pVCpu);
6290
6291 return VINF_SUCCESS;
6292}
6293
6294
6295/**
6296 * Saves the auto load/store'd guest MSRs from the current VMCS into
6297 * the guest-CPU context.
6298 *
6299 * @returns VBox status code.
6300 * @param pVCpu The cross context virtual CPU structure.
6301 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6302 * out-of-sync. Make sure to update the required fields
6303 * before using them.
6304 *
6305 * @remarks No-long-jump zone!!!
6306 */
6307static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6308{
6309 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6310 return VINF_SUCCESS;
6311
6312 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6313 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6314 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6315 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6316 {
6317 switch (pMsr->u32Msr)
6318 {
6319 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6320 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6321 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6322 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6323 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6324 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6325 break;
6326
6327 default:
6328 {
6329 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6330 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6331 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6332 }
6333 }
6334 }
6335
6336 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6337 return VINF_SUCCESS;
6338}
6339
6340
6341/**
6342 * Saves the guest control registers from the current VMCS into the guest-CPU
6343 * context.
6344 *
6345 * @returns VBox status code.
6346 * @param pVCpu The cross context virtual CPU structure.
6347 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6348 * out-of-sync. Make sure to update the required fields
6349 * before using them.
6350 *
6351 * @remarks No-long-jump zone!!!
6352 */
6353static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6354{
6355 /* Guest CR0. Guest FPU. */
6356 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6357 AssertRCReturn(rc, rc);
6358
6359 /* Guest CR4. */
6360 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6361 AssertRCReturn(rc, rc);
6362
6363 /* Guest CR2 - updated always during the world-switch or in #PF. */
6364 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6365 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6366 {
6367 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6368 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6369
6370 PVM pVM = pVCpu->CTX_SUFF(pVM);
6371 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6372 || ( pVM->hm.s.fNestedPaging
6373 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6374 {
6375 uint64_t u64Val = 0;
6376 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6377 if (pMixedCtx->cr3 != u64Val)
6378 {
6379 CPUMSetGuestCR3(pVCpu, u64Val);
6380 if (VMMRZCallRing3IsEnabled(pVCpu))
6381 {
6382 PGMUpdateCR3(pVCpu, u64Val);
6383 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6384 }
6385 else
6386 {
6387 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6388 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6389 }
6390 }
6391
6392 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6393 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6394 {
6395 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6396 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6397 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6398 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6399 AssertRCReturn(rc, rc);
6400
6401 if (VMMRZCallRing3IsEnabled(pVCpu))
6402 {
6403 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6404 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6405 }
6406 else
6407 {
6408 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6409 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6410 }
6411 }
6412 }
6413
6414 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6415 }
6416
6417 /*
6418 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6419 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6420 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6421 *
6422 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6423 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6424 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6425 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6426 *
6427 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6428 */
6429 if (VMMRZCallRing3IsEnabled(pVCpu))
6430 {
6431 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6432 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6433
6434 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6435 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6436
6437 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6438 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6439 }
6440
6441 return rc;
6442}
6443
6444
6445/**
6446 * Reads a guest segment register from the current VMCS into the guest-CPU
6447 * context.
6448 *
6449 * @returns VBox status code.
6450 * @param pVCpu The cross context virtual CPU structure.
6451 * @param idxSel Index of the selector in the VMCS.
6452 * @param idxLimit Index of the segment limit in the VMCS.
6453 * @param idxBase Index of the segment base in the VMCS.
6454 * @param idxAccess Index of the access rights of the segment in the VMCS.
6455 * @param pSelReg Pointer to the segment selector.
6456 *
6457 * @remarks No-long-jump zone!!!
6458 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6459 * macro as that takes care of whether to read from the VMCS cache or
6460 * not.
6461 */
6462DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6463 PCPUMSELREG pSelReg)
6464{
6465 NOREF(pVCpu);
6466
6467 uint32_t u32Val = 0;
6468 int rc = VMXReadVmcs32(idxSel, &u32Val);
6469 AssertRCReturn(rc, rc);
6470 pSelReg->Sel = (uint16_t)u32Val;
6471 pSelReg->ValidSel = (uint16_t)u32Val;
6472 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6473
6474 rc = VMXReadVmcs32(idxLimit, &u32Val);
6475 AssertRCReturn(rc, rc);
6476 pSelReg->u32Limit = u32Val;
6477
6478 uint64_t u64Val = 0;
6479 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6480 AssertRCReturn(rc, rc);
6481 pSelReg->u64Base = u64Val;
6482
6483 rc = VMXReadVmcs32(idxAccess, &u32Val);
6484 AssertRCReturn(rc, rc);
6485 pSelReg->Attr.u = u32Val;
6486
6487 /*
6488 * If VT-x marks the segment as unusable, most other bits remain undefined:
6489 * - For CS the L, D and G bits have meaning.
6490 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6491 * - For the remaining data segments no bits are defined.
6492 *
6493 * The present bit and the unusable bit has been observed to be set at the
6494 * same time (the selector was supposed to be invalid as we started executing
6495 * a V8086 interrupt in ring-0).
6496 *
6497 * What should be important for the rest of the VBox code, is that the P bit is
6498 * cleared. Some of the other VBox code recognizes the unusable bit, but
6499 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6500 * safe side here, we'll strip off P and other bits we don't care about. If
6501 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6502 *
6503 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6504 */
6505 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6506 {
6507 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6508
6509 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6510 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6511 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6512
6513 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6514#ifdef DEBUG_bird
6515 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6516 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6517 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6518#endif
6519 }
6520 return VINF_SUCCESS;
6521}
6522
6523
6524#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6525# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6526 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6527 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6528#else
6529# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6530 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6531 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6532#endif
6533
6534
6535/**
6536 * Saves the guest segment registers from the current VMCS into the guest-CPU
6537 * context.
6538 *
6539 * @returns VBox status code.
6540 * @param pVCpu The cross context virtual CPU structure.
6541 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6542 * out-of-sync. Make sure to update the required fields
6543 * before using them.
6544 *
6545 * @remarks No-long-jump zone!!!
6546 */
6547static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6548{
6549 /* Guest segment registers. */
6550 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6551 {
6552 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6553 AssertRCReturn(rc, rc);
6554
6555 rc = VMXLOCAL_READ_SEG(CS, cs);
6556 rc |= VMXLOCAL_READ_SEG(SS, ss);
6557 rc |= VMXLOCAL_READ_SEG(DS, ds);
6558 rc |= VMXLOCAL_READ_SEG(ES, es);
6559 rc |= VMXLOCAL_READ_SEG(FS, fs);
6560 rc |= VMXLOCAL_READ_SEG(GS, gs);
6561 AssertRCReturn(rc, rc);
6562
6563 /* Restore segment attributes for real-on-v86 mode hack. */
6564 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6565 {
6566 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6567 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6568 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6569 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6570 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6571 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6572 }
6573 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6574 }
6575
6576 return VINF_SUCCESS;
6577}
6578
6579
6580/**
6581 * Saves the guest descriptor table registers and task register from the current
6582 * VMCS into the guest-CPU context.
6583 *
6584 * @returns VBox status code.
6585 * @param pVCpu The cross context virtual CPU structure.
6586 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6587 * out-of-sync. Make sure to update the required fields
6588 * before using them.
6589 *
6590 * @remarks No-long-jump zone!!!
6591 */
6592static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6593{
6594 int rc = VINF_SUCCESS;
6595
6596 /* Guest LDTR. */
6597 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6598 {
6599 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6600 AssertRCReturn(rc, rc);
6601 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6602 }
6603
6604 /* Guest GDTR. */
6605 uint64_t u64Val = 0;
6606 uint32_t u32Val = 0;
6607 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6608 {
6609 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6610 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6611 pMixedCtx->gdtr.pGdt = u64Val;
6612 pMixedCtx->gdtr.cbGdt = u32Val;
6613 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6614 }
6615
6616 /* Guest IDTR. */
6617 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6618 {
6619 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6620 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6621 pMixedCtx->idtr.pIdt = u64Val;
6622 pMixedCtx->idtr.cbIdt = u32Val;
6623 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6624 }
6625
6626 /* Guest TR. */
6627 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6628 {
6629 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6630 AssertRCReturn(rc, rc);
6631
6632 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6633 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6634 {
6635 rc = VMXLOCAL_READ_SEG(TR, tr);
6636 AssertRCReturn(rc, rc);
6637 }
6638 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6639 }
6640 return rc;
6641}
6642
6643#undef VMXLOCAL_READ_SEG
6644
6645
6646/**
6647 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6648 * context.
6649 *
6650 * @returns VBox status code.
6651 * @param pVCpu The cross context virtual CPU structure.
6652 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6653 * out-of-sync. Make sure to update the required fields
6654 * before using them.
6655 *
6656 * @remarks No-long-jump zone!!!
6657 */
6658static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6659{
6660 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6661 {
6662 if (!pVCpu->hm.s.fUsingHyperDR7)
6663 {
6664 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6665 uint32_t u32Val;
6666 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6667 pMixedCtx->dr[7] = u32Val;
6668 }
6669
6670 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6671 }
6672 return VINF_SUCCESS;
6673}
6674
6675
6676/**
6677 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6678 *
6679 * @returns VBox status code.
6680 * @param pVCpu The cross context virtual CPU structure.
6681 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6682 * out-of-sync. Make sure to update the required fields
6683 * before using them.
6684 *
6685 * @remarks No-long-jump zone!!!
6686 */
6687static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6688{
6689 NOREF(pMixedCtx);
6690
6691 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6692 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6693 return VINF_SUCCESS;
6694}
6695
6696
6697/**
6698 * Saves the entire guest state from the currently active VMCS into the
6699 * guest-CPU context.
6700 *
6701 * This essentially VMREADs all guest-data.
6702 *
6703 * @returns VBox status code.
6704 * @param pVCpu The cross context virtual CPU structure.
6705 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6706 * out-of-sync. Make sure to update the required fields
6707 * before using them.
6708 */
6709static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6710{
6711 Assert(pVCpu);
6712 Assert(pMixedCtx);
6713
6714 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6715 return VINF_SUCCESS;
6716
6717 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6718 again on the ring-3 callback path, there is no real need to. */
6719 if (VMMRZCallRing3IsEnabled(pVCpu))
6720 VMMR0LogFlushDisable(pVCpu);
6721 else
6722 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6723 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6724
6725 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6726 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6727
6728 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6729 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6730
6731 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6732 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6733
6734 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6735 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6736
6737 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6738 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6739
6740 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6741 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6742
6743 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6744 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6745
6746 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6747 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6748
6749 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6750 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6751
6752 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6753 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6754
6755 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6756 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6757 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6758
6759 if (VMMRZCallRing3IsEnabled(pVCpu))
6760 VMMR0LogFlushEnable(pVCpu);
6761
6762 return VINF_SUCCESS;
6763}
6764
6765
6766/**
6767 * Saves basic guest registers needed for IEM instruction execution.
6768 *
6769 * @returns VBox status code (OR-able).
6770 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6771 * @param pMixedCtx Pointer to the CPU context of the guest.
6772 * @param fMemory Whether the instruction being executed operates on
6773 * memory or not. Only CR0 is synced up if clear.
6774 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6775 */
6776static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6777{
6778 /*
6779 * We assume all general purpose registers other than RSP are available.
6780 *
6781 * RIP is a must, as it will be incremented or otherwise changed.
6782 *
6783 * RFLAGS are always required to figure the CPL.
6784 *
6785 * RSP isn't always required, however it's a GPR, so frequently required.
6786 *
6787 * SS and CS are the only segment register needed if IEM doesn't do memory
6788 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6789 *
6790 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6791 * be required for memory accesses.
6792 *
6793 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6794 */
6795 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6796 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6797 if (fNeedRsp)
6798 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6799 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6800 if (!fMemory)
6801 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6802 else
6803 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6804 AssertRCReturn(rc, rc);
6805 return rc;
6806}
6807
6808
6809/**
6810 * Ensures that we've got a complete basic guest-context.
6811 *
6812 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6813 * is for the interpreter.
6814 *
6815 * @returns VBox status code.
6816 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6817 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6818 * needing to be synced in.
6819 * @thread EMT(pVCpu)
6820 */
6821VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6822{
6823 /* Note! Since this is only applicable to VT-x, the implementation is placed
6824 in the VT-x part of the sources instead of the generic stuff. */
6825 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6826 {
6827 /* For now, imply that the caller might change everything too. */
6828 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6829 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6830 }
6831 return VINF_SUCCESS;
6832}
6833
6834
6835/**
6836 * Check per-VM and per-VCPU force flag actions that require us to go back to
6837 * ring-3 for one reason or another.
6838 *
6839 * @returns Strict VBox status code (i.e. informational status codes too)
6840 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6841 * ring-3.
6842 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6843 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6844 * interrupts)
6845 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6846 * all EMTs to be in ring-3.
6847 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6848 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6849 * to the EM loop.
6850 *
6851 * @param pVM The cross context VM structure.
6852 * @param pVCpu The cross context virtual CPU structure.
6853 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6854 * out-of-sync. Make sure to update the required fields
6855 * before using them.
6856 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6857 */
6858static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6859{
6860 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6861
6862 /*
6863 * Anything pending? Should be more likely than not if we're doing a good job.
6864 */
6865 if ( !fStepping
6866 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6867 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6868 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6869 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6870 return VINF_SUCCESS;
6871
6872 /* We need the control registers now, make sure the guest-CPU context is updated. */
6873 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6874 AssertRCReturn(rc3, rc3);
6875
6876 /* Pending HM CR3 sync. */
6877 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6878 {
6879 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6880 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6881 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6882 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6883 }
6884
6885 /* Pending HM PAE PDPEs. */
6886 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6887 {
6888 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6889 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6890 }
6891
6892 /* Pending PGM C3 sync. */
6893 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6894 {
6895 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6896 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6897 if (rcStrict2 != VINF_SUCCESS)
6898 {
6899 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6900 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6901 return rcStrict2;
6902 }
6903 }
6904
6905 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6906 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6907 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6908 {
6909 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6910 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6911 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6912 return rc2;
6913 }
6914
6915 /* Pending VM request packets, such as hardware interrupts. */
6916 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6917 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6918 {
6919 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6920 return VINF_EM_PENDING_REQUEST;
6921 }
6922
6923 /* Pending PGM pool flushes. */
6924 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6925 {
6926 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6927 return VINF_PGM_POOL_FLUSH_PENDING;
6928 }
6929
6930 /* Pending DMA requests. */
6931 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6932 {
6933 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6934 return VINF_EM_RAW_TO_R3;
6935 }
6936
6937 return VINF_SUCCESS;
6938}
6939
6940
6941/**
6942 * Converts any TRPM trap into a pending HM event. This is typically used when
6943 * entering from ring-3 (not longjmp returns).
6944 *
6945 * @param pVCpu The cross context virtual CPU structure.
6946 */
6947static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6948{
6949 Assert(TRPMHasTrap(pVCpu));
6950 Assert(!pVCpu->hm.s.Event.fPending);
6951
6952 uint8_t uVector;
6953 TRPMEVENT enmTrpmEvent;
6954 RTGCUINT uErrCode;
6955 RTGCUINTPTR GCPtrFaultAddress;
6956 uint8_t cbInstr;
6957
6958 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6959 AssertRC(rc);
6960
6961 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6962 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6963 if (enmTrpmEvent == TRPM_TRAP)
6964 {
6965 switch (uVector)
6966 {
6967 case X86_XCPT_NMI:
6968 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6969 break;
6970
6971 case X86_XCPT_BP:
6972 case X86_XCPT_OF:
6973 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6974 break;
6975
6976 case X86_XCPT_PF:
6977 case X86_XCPT_DF:
6978 case X86_XCPT_TS:
6979 case X86_XCPT_NP:
6980 case X86_XCPT_SS:
6981 case X86_XCPT_GP:
6982 case X86_XCPT_AC:
6983 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6984 /* no break! */
6985 default:
6986 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6987 break;
6988 }
6989 }
6990 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6991 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6992 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6993 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6994 else
6995 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6996
6997 rc = TRPMResetTrap(pVCpu);
6998 AssertRC(rc);
6999 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7000 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7001
7002 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7003}
7004
7005
7006/**
7007 * Converts the pending HM event into a TRPM trap.
7008 *
7009 * @param pVCpu The cross context virtual CPU structure.
7010 */
7011static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7012{
7013 Assert(pVCpu->hm.s.Event.fPending);
7014
7015 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7016 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7017 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7018 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7019
7020 /* If a trap was already pending, we did something wrong! */
7021 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7022
7023 TRPMEVENT enmTrapType;
7024 switch (uVectorType)
7025 {
7026 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7027 enmTrapType = TRPM_HARDWARE_INT;
7028 break;
7029
7030 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7031 enmTrapType = TRPM_SOFTWARE_INT;
7032 break;
7033
7034 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7035 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7036 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7037 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7038 enmTrapType = TRPM_TRAP;
7039 break;
7040
7041 default:
7042 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7043 enmTrapType = TRPM_32BIT_HACK;
7044 break;
7045 }
7046
7047 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7048
7049 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7050 AssertRC(rc);
7051
7052 if (fErrorCodeValid)
7053 TRPMSetErrorCode(pVCpu, uErrorCode);
7054
7055 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7056 && uVector == X86_XCPT_PF)
7057 {
7058 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7059 }
7060 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7061 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7062 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7063 {
7064 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7065 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7066 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7067 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7068 }
7069
7070 /* Clear any pending events from the VMCS. */
7071 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7072 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7073
7074 /* We're now done converting the pending event. */
7075 pVCpu->hm.s.Event.fPending = false;
7076}
7077
7078
7079/**
7080 * Does the necessary state syncing before returning to ring-3 for any reason
7081 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7082 *
7083 * @returns VBox status code.
7084 * @param pVM The cross context VM structure.
7085 * @param pVCpu The cross context virtual CPU structure.
7086 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7087 * be out-of-sync. Make sure to update the required
7088 * fields before using them.
7089 * @param fSaveGuestState Whether to save the guest state or not.
7090 *
7091 * @remarks No-long-jmp zone!!!
7092 */
7093static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7094{
7095 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7096 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7097
7098 RTCPUID idCpu = RTMpCpuId();
7099 Log4Func(("HostCpuId=%u\n", idCpu));
7100
7101 /*
7102 * !!! IMPORTANT !!!
7103 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7104 */
7105
7106 /* Save the guest state if necessary. */
7107 if ( fSaveGuestState
7108 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7109 {
7110 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7111 AssertRCReturn(rc, rc);
7112 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7113 }
7114
7115 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7116 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7117 {
7118 if (fSaveGuestState)
7119 {
7120 /* We shouldn't reload CR0 without saving it first. */
7121 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7122 AssertRCReturn(rc, rc);
7123 }
7124 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7125 }
7126
7127 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7128#ifdef VBOX_STRICT
7129 if (CPUMIsHyperDebugStateActive(pVCpu))
7130 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7131#endif
7132 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7133 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7134 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7135 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7136
7137#if HC_ARCH_BITS == 64
7138 /* Restore host-state bits that VT-x only restores partially. */
7139 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7140 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7141 {
7142 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7143 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7144 }
7145 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7146#endif
7147
7148 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7149 if (pVCpu->hm.s.vmx.fLazyMsrs)
7150 {
7151 /* We shouldn't reload the guest MSRs without saving it first. */
7152 if (!fSaveGuestState)
7153 {
7154 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7155 AssertRCReturn(rc, rc);
7156 }
7157 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7158 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7159 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7160 }
7161
7162 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7163 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7164
7165 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7166 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7167 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7168 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7169 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7170 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7171 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7172 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7173
7174 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7175
7176 /** @todo This partially defeats the purpose of having preemption hooks.
7177 * The problem is, deregistering the hooks should be moved to a place that
7178 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7179 * context.
7180 */
7181 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7182 {
7183 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7184 AssertRCReturn(rc, rc);
7185
7186 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7187 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7188 }
7189 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7190 NOREF(idCpu);
7191
7192 return VINF_SUCCESS;
7193}
7194
7195
7196/**
7197 * Leaves the VT-x session.
7198 *
7199 * @returns VBox status code.
7200 * @param pVM The cross context VM structure.
7201 * @param pVCpu The cross context virtual CPU structure.
7202 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7203 * out-of-sync. Make sure to update the required fields
7204 * before using them.
7205 *
7206 * @remarks No-long-jmp zone!!!
7207 */
7208DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7209{
7210 HM_DISABLE_PREEMPT();
7211 HMVMX_ASSERT_CPU_SAFE();
7212 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7213 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7214
7215 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7216 and done this from the VMXR0ThreadCtxCallback(). */
7217 if (!pVCpu->hm.s.fLeaveDone)
7218 {
7219 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7220 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7221 pVCpu->hm.s.fLeaveDone = true;
7222 }
7223 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7224
7225 /*
7226 * !!! IMPORTANT !!!
7227 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7228 */
7229
7230 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7231 /** @todo Deregistering here means we need to VMCLEAR always
7232 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7233 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7234 VMMR0ThreadCtxHookDisable(pVCpu);
7235
7236 /* Leave HM context. This takes care of local init (term). */
7237 int rc = HMR0LeaveCpu(pVCpu);
7238
7239 HM_RESTORE_PREEMPT();
7240 return rc;
7241}
7242
7243
7244/**
7245 * Does the necessary state syncing before doing a longjmp to ring-3.
7246 *
7247 * @returns VBox status code.
7248 * @param pVM The cross context VM structure.
7249 * @param pVCpu The cross context virtual CPU structure.
7250 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7251 * out-of-sync. Make sure to update the required fields
7252 * before using them.
7253 *
7254 * @remarks No-long-jmp zone!!!
7255 */
7256DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7257{
7258 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7259}
7260
7261
7262/**
7263 * Take necessary actions before going back to ring-3.
7264 *
7265 * An action requires us to go back to ring-3. This function does the necessary
7266 * steps before we can safely return to ring-3. This is not the same as longjmps
7267 * to ring-3, this is voluntary and prepares the guest so it may continue
7268 * executing outside HM (recompiler/IEM).
7269 *
7270 * @returns VBox status code.
7271 * @param pVM The cross context VM structure.
7272 * @param pVCpu The cross context virtual CPU structure.
7273 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7274 * out-of-sync. Make sure to update the required fields
7275 * before using them.
7276 * @param rcExit The reason for exiting to ring-3. Can be
7277 * VINF_VMM_UNKNOWN_RING3_CALL.
7278 */
7279static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7280{
7281 Assert(pVM);
7282 Assert(pVCpu);
7283 Assert(pMixedCtx);
7284 HMVMX_ASSERT_PREEMPT_SAFE();
7285
7286 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7287 {
7288 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7289 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7290 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7291 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7292 }
7293
7294 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7295 VMMRZCallRing3Disable(pVCpu);
7296 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7297
7298 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7299 if (pVCpu->hm.s.Event.fPending)
7300 {
7301 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7302 Assert(!pVCpu->hm.s.Event.fPending);
7303 }
7304
7305 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7306 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7307
7308 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7309 and if we're injecting an event we should have a TRPM trap pending. */
7310 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7311#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7312 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7313#endif
7314
7315 /* Save guest state and restore host state bits. */
7316 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7317 AssertRCReturn(rc, rc);
7318 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7319 /* Thread-context hooks are unregistered at this point!!! */
7320
7321 /* Sync recompiler state. */
7322 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7323 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7324 | CPUM_CHANGED_LDTR
7325 | CPUM_CHANGED_GDTR
7326 | CPUM_CHANGED_IDTR
7327 | CPUM_CHANGED_TR
7328 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7329 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7330 if ( pVM->hm.s.fNestedPaging
7331 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7332 {
7333 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7334 }
7335
7336 Assert(!pVCpu->hm.s.fClearTrapFlag);
7337
7338 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7339 if (rcExit != VINF_EM_RAW_INTERRUPT)
7340 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7341
7342 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7343
7344 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7345 VMMRZCallRing3RemoveNotification(pVCpu);
7346 VMMRZCallRing3Enable(pVCpu);
7347
7348 return rc;
7349}
7350
7351
7352/**
7353 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7354 * longjump to ring-3 and possibly get preempted.
7355 *
7356 * @returns VBox status code.
7357 * @param pVCpu The cross context virtual CPU structure.
7358 * @param enmOperation The operation causing the ring-3 longjump.
7359 * @param pvUser Opaque pointer to the guest-CPU context. The data
7360 * may be out-of-sync. Make sure to update the required
7361 * fields before using them.
7362 */
7363static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7364{
7365 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7366 {
7367 /*
7368 * !!! IMPORTANT !!!
7369 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7370 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7371 */
7372 VMMRZCallRing3RemoveNotification(pVCpu);
7373 VMMRZCallRing3Disable(pVCpu);
7374 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7375 RTThreadPreemptDisable(&PreemptState);
7376
7377 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7378 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7379
7380#if HC_ARCH_BITS == 64
7381 /* Restore host-state bits that VT-x only restores partially. */
7382 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7383 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7384 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7385 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7386#endif
7387 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7388 if (pVCpu->hm.s.vmx.fLazyMsrs)
7389 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7390
7391 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7392 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7393 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7394 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7395 {
7396 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7397 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7398 }
7399
7400 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7401 VMMR0ThreadCtxHookDisable(pVCpu);
7402 HMR0LeaveCpu(pVCpu);
7403 RTThreadPreemptRestore(&PreemptState);
7404 return VINF_SUCCESS;
7405 }
7406
7407 Assert(pVCpu);
7408 Assert(pvUser);
7409 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7410 HMVMX_ASSERT_PREEMPT_SAFE();
7411
7412 VMMRZCallRing3Disable(pVCpu);
7413 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7414
7415 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7416 enmOperation));
7417
7418 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7419 AssertRCReturn(rc, rc);
7420
7421 VMMRZCallRing3Enable(pVCpu);
7422 return VINF_SUCCESS;
7423}
7424
7425
7426/**
7427 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7428 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7429 *
7430 * @param pVCpu The cross context virtual CPU structure.
7431 */
7432DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7433{
7434 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7435 {
7436 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7437 {
7438 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7439 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7440 AssertRC(rc);
7441 Log4(("Setup interrupt-window exiting\n"));
7442 }
7443 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7444}
7445
7446
7447/**
7448 * Clears the interrupt-window exiting control in the VMCS.
7449 *
7450 * @param pVCpu The cross context virtual CPU structure.
7451 */
7452DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7453{
7454 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7455 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7456 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7457 AssertRC(rc);
7458 Log4(("Cleared interrupt-window exiting\n"));
7459}
7460
7461
7462/**
7463 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7464 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7465 *
7466 * @param pVCpu The cross context virtual CPU structure.
7467 */
7468DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7469{
7470 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7471 {
7472 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7473 {
7474 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7475 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7476 AssertRC(rc);
7477 Log4(("Setup NMI-window exiting\n"));
7478 }
7479 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7480}
7481
7482
7483/**
7484 * Clears the NMI-window exiting control in the VMCS.
7485 *
7486 * @param pVCpu The cross context virtual CPU structure.
7487 */
7488DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7489{
7490 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7491 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7492 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7493 AssertRC(rc);
7494 Log4(("Cleared NMI-window exiting\n"));
7495}
7496
7497
7498/**
7499 * Evaluates the event to be delivered to the guest and sets it as the pending
7500 * event.
7501 *
7502 * @returns The VT-x guest-interruptibility state.
7503 * @param pVCpu The cross context virtual CPU structure.
7504 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7505 * out-of-sync. Make sure to update the required fields
7506 * before using them.
7507 */
7508static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7509{
7510 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7511 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7512 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7513 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7514 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7515
7516 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7517 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7518 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7519 Assert(!TRPMHasTrap(pVCpu));
7520
7521#ifdef VBOX_WITH_NEW_APIC
7522 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7523 APICUpdatePendingInterrupts(pVCpu);
7524#endif
7525
7526 /*
7527 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7528 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7529 */
7530 /** @todo SMI. SMIs take priority over NMIs. */
7531 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7532 {
7533 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7534 if ( !pVCpu->hm.s.Event.fPending
7535 && !fBlockNmi
7536 && !fBlockSti
7537 && !fBlockMovSS)
7538 {
7539 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7540 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7541 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7542
7543 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7544 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7545 }
7546 else
7547 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7548 }
7549 /*
7550 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7551 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7552 */
7553 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7554 && !pVCpu->hm.s.fSingleInstruction)
7555 {
7556 Assert(!DBGFIsStepping(pVCpu));
7557 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7558 AssertRC(rc);
7559 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7560 if ( !pVCpu->hm.s.Event.fPending
7561 && !fBlockInt
7562 && !fBlockSti
7563 && !fBlockMovSS)
7564 {
7565 uint8_t u8Interrupt;
7566 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7567 if (RT_SUCCESS(rc))
7568 {
7569 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7570 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7571 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7572
7573 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7574 }
7575 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7576 {
7577 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7578 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7579 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7580 }
7581 else
7582 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7583 }
7584 else
7585 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7586 }
7587
7588 return uIntrState;
7589}
7590
7591
7592/**
7593 * Sets a pending-debug exception to be delivered to the guest if the guest is
7594 * single-stepping in the VMCS.
7595 *
7596 * @param pVCpu The cross context virtual CPU structure.
7597 */
7598DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7599{
7600 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7601 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7602 AssertRC(rc);
7603}
7604
7605
7606/**
7607 * Injects any pending events into the guest if the guest is in a state to
7608 * receive them.
7609 *
7610 * @returns Strict VBox status code (i.e. informational status codes too).
7611 * @param pVCpu The cross context virtual CPU structure.
7612 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7613 * out-of-sync. Make sure to update the required fields
7614 * before using them.
7615 * @param uIntrState The VT-x guest-interruptibility state.
7616 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7617 * return VINF_EM_DBG_STEPPED if the event was
7618 * dispatched directly.
7619 */
7620static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7621{
7622 HMVMX_ASSERT_PREEMPT_SAFE();
7623 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7624
7625 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7626 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7627
7628 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7629 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7630 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7631 Assert(!TRPMHasTrap(pVCpu));
7632
7633 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7634 if (pVCpu->hm.s.Event.fPending)
7635 {
7636 /*
7637 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7638 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7639 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7640 *
7641 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7642 */
7643 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7644#ifdef VBOX_STRICT
7645 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7646 {
7647 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7648 Assert(!fBlockInt);
7649 Assert(!fBlockSti);
7650 Assert(!fBlockMovSS);
7651 }
7652 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7653 {
7654 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7655 Assert(!fBlockSti);
7656 Assert(!fBlockMovSS);
7657 Assert(!fBlockNmi);
7658 }
7659#endif
7660 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7661 (uint8_t)uIntType));
7662 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7663 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7664 fStepping, &uIntrState);
7665 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7666
7667 /* Update the interruptibility-state as it could have been changed by
7668 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7669 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7670 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7671
7672 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7673 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7674 else
7675 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7676 }
7677
7678 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7679 if ( fBlockSti
7680 || fBlockMovSS)
7681 {
7682 if (!pVCpu->hm.s.fSingleInstruction)
7683 {
7684 /*
7685 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7686 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7687 * See Intel spec. 27.3.4 "Saving Non-Register State".
7688 */
7689 Assert(!DBGFIsStepping(pVCpu));
7690 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7691 AssertRCReturn(rc2, rc2);
7692 if (pMixedCtx->eflags.Bits.u1TF)
7693 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7694 }
7695 else if (pMixedCtx->eflags.Bits.u1TF)
7696 {
7697 /*
7698 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7699 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7700 */
7701 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7702 uIntrState = 0;
7703 }
7704 }
7705
7706 /*
7707 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7708 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7709 */
7710 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7711 AssertRC(rc2);
7712
7713 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7714 NOREF(fBlockMovSS); NOREF(fBlockSti);
7715 return rcStrict;
7716}
7717
7718
7719/**
7720 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7721 *
7722 * @param pVCpu The cross context virtual CPU structure.
7723 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7724 * out-of-sync. Make sure to update the required fields
7725 * before using them.
7726 */
7727DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7728{
7729 NOREF(pMixedCtx);
7730 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7731 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7732}
7733
7734
7735/**
7736 * Injects a double-fault (\#DF) exception into the VM.
7737 *
7738 * @returns Strict VBox status code (i.e. informational status codes too).
7739 * @param pVCpu The cross context virtual CPU structure.
7740 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7741 * out-of-sync. Make sure to update the required fields
7742 * before using them.
7743 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7744 * and should return VINF_EM_DBG_STEPPED if the event
7745 * is injected directly (register modified by us, not
7746 * by hardware on VM-entry).
7747 * @param puIntrState Pointer to the current guest interruptibility-state.
7748 * This interruptibility-state will be updated if
7749 * necessary. This cannot not be NULL.
7750 */
7751DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7752{
7753 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7754 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7755 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7756 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7757 fStepping, puIntrState);
7758}
7759
7760
7761/**
7762 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7763 *
7764 * @param pVCpu The cross context virtual CPU structure.
7765 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7766 * out-of-sync. Make sure to update the required fields
7767 * before using them.
7768 */
7769DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7770{
7771 NOREF(pMixedCtx);
7772 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7773 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7774 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7775}
7776
7777
7778/**
7779 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7780 *
7781 * @param pVCpu The cross context virtual CPU structure.
7782 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7783 * out-of-sync. Make sure to update the required fields
7784 * before using them.
7785 * @param cbInstr The value of RIP that is to be pushed on the guest
7786 * stack.
7787 */
7788DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7789{
7790 NOREF(pMixedCtx);
7791 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7792 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7793 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7794}
7795
7796
7797/**
7798 * Injects a general-protection (\#GP) fault into the VM.
7799 *
7800 * @returns Strict VBox status code (i.e. informational status codes too).
7801 * @param pVCpu The cross context virtual CPU structure.
7802 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7803 * out-of-sync. Make sure to update the required fields
7804 * before using them.
7805 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7806 * mode, i.e. in real-mode it's not valid).
7807 * @param u32ErrorCode The error code associated with the \#GP.
7808 * @param fStepping Whether we're running in
7809 * hmR0VmxRunGuestCodeStep() and should return
7810 * VINF_EM_DBG_STEPPED if the event is injected
7811 * directly (register modified by us, not by
7812 * hardware on VM-entry).
7813 * @param puIntrState Pointer to the current guest interruptibility-state.
7814 * This interruptibility-state will be updated if
7815 * necessary. This cannot not be NULL.
7816 */
7817DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7818 bool fStepping, uint32_t *puIntrState)
7819{
7820 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7821 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7822 if (fErrorCodeValid)
7823 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7824 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7825 fStepping, puIntrState);
7826}
7827
7828
7829/**
7830 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7831 * VM.
7832 *
7833 * @param pVCpu The cross context virtual CPU structure.
7834 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7835 * out-of-sync. Make sure to update the required fields
7836 * before using them.
7837 * @param u32ErrorCode The error code associated with the \#GP.
7838 */
7839DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7840{
7841 NOREF(pMixedCtx);
7842 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7843 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7844 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7845 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7846}
7847
7848
7849/**
7850 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7851 *
7852 * @param pVCpu The cross context virtual CPU structure.
7853 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7854 * out-of-sync. Make sure to update the required fields
7855 * before using them.
7856 * @param uVector The software interrupt vector number.
7857 * @param cbInstr The value of RIP that is to be pushed on the guest
7858 * stack.
7859 */
7860DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7861{
7862 NOREF(pMixedCtx);
7863 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7864 if ( uVector == X86_XCPT_BP
7865 || uVector == X86_XCPT_OF)
7866 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7867 else
7868 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7869 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7870}
7871
7872
7873/**
7874 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7875 * stack.
7876 *
7877 * @returns Strict VBox status code (i.e. informational status codes too).
7878 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7879 * @param pVM The cross context VM structure.
7880 * @param pMixedCtx Pointer to the guest-CPU context.
7881 * @param uValue The value to push to the guest stack.
7882 */
7883DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7884{
7885 /*
7886 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7887 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7888 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7889 */
7890 if (pMixedCtx->sp == 1)
7891 return VINF_EM_RESET;
7892 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7893 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7894 AssertRC(rc);
7895 return rc;
7896}
7897
7898
7899/**
7900 * Injects an event into the guest upon VM-entry by updating the relevant fields
7901 * in the VM-entry area in the VMCS.
7902 *
7903 * @returns Strict VBox status code (i.e. informational status codes too).
7904 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7905 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7906 *
7907 * @param pVCpu The cross context virtual CPU structure.
7908 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7909 * be out-of-sync. Make sure to update the required
7910 * fields before using them.
7911 * @param u64IntInfo The VM-entry interruption-information field.
7912 * @param cbInstr The VM-entry instruction length in bytes (for
7913 * software interrupts, exceptions and privileged
7914 * software exceptions).
7915 * @param u32ErrCode The VM-entry exception error code.
7916 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7917 * @param puIntrState Pointer to the current guest interruptibility-state.
7918 * This interruptibility-state will be updated if
7919 * necessary. This cannot not be NULL.
7920 * @param fStepping Whether we're running in
7921 * hmR0VmxRunGuestCodeStep() and should return
7922 * VINF_EM_DBG_STEPPED if the event is injected
7923 * directly (register modified by us, not by
7924 * hardware on VM-entry).
7925 *
7926 * @remarks Requires CR0!
7927 * @remarks No-long-jump zone!!!
7928 */
7929static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7930 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7931 uint32_t *puIntrState)
7932{
7933 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7934 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7935 Assert(puIntrState);
7936 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7937
7938 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7939 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7940
7941#ifdef VBOX_STRICT
7942 /* Validate the error-code-valid bit for hardware exceptions. */
7943 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7944 {
7945 switch (uVector)
7946 {
7947 case X86_XCPT_PF:
7948 case X86_XCPT_DF:
7949 case X86_XCPT_TS:
7950 case X86_XCPT_NP:
7951 case X86_XCPT_SS:
7952 case X86_XCPT_GP:
7953 case X86_XCPT_AC:
7954 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7955 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7956 /* fallthru */
7957 default:
7958 break;
7959 }
7960 }
7961#endif
7962
7963 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7964 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7965 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7966
7967 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7968
7969 /* We require CR0 to check if the guest is in real-mode. */
7970 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7971 AssertRCReturn(rc, rc);
7972
7973 /*
7974 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7975 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7976 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7977 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7978 */
7979 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7980 {
7981 PVM pVM = pVCpu->CTX_SUFF(pVM);
7982 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7983 {
7984 Assert(PDMVmmDevHeapIsEnabled(pVM));
7985 Assert(pVM->hm.s.vmx.pRealModeTSS);
7986
7987 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7988 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7989 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7990 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7991 AssertRCReturn(rc, rc);
7992 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7993
7994 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7995 size_t const cbIdtEntry = sizeof(X86IDTR16);
7996 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7997 {
7998 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7999 if (uVector == X86_XCPT_DF)
8000 return VINF_EM_RESET;
8001
8002 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8003 if (uVector == X86_XCPT_GP)
8004 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8005
8006 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8007 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8008 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8009 fStepping, puIntrState);
8010 }
8011
8012 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8013 uint16_t uGuestIp = pMixedCtx->ip;
8014 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8015 {
8016 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8017 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8018 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8019 }
8020 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8021 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8022
8023 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8024 X86IDTR16 IdtEntry;
8025 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8026 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8027 AssertRCReturn(rc, rc);
8028
8029 /* Construct the stack frame for the interrupt/exception handler. */
8030 VBOXSTRICTRC rcStrict;
8031 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8032 if (rcStrict == VINF_SUCCESS)
8033 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8034 if (rcStrict == VINF_SUCCESS)
8035 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8036
8037 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8038 if (rcStrict == VINF_SUCCESS)
8039 {
8040 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8041 pMixedCtx->rip = IdtEntry.offSel;
8042 pMixedCtx->cs.Sel = IdtEntry.uSel;
8043 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8044 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8045 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8046 && uVector == X86_XCPT_PF)
8047 pMixedCtx->cr2 = GCPtrFaultAddress;
8048
8049 /* If any other guest-state bits are changed here, make sure to update
8050 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8051 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8052 | HM_CHANGED_GUEST_RIP
8053 | HM_CHANGED_GUEST_RFLAGS
8054 | HM_CHANGED_GUEST_RSP);
8055
8056 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8057 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8058 {
8059 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8060 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8061 Log4(("Clearing inhibition due to STI.\n"));
8062 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8063 }
8064 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8065 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8066
8067 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8068 it, if we are returning to ring-3 before executing guest code. */
8069 pVCpu->hm.s.Event.fPending = false;
8070
8071 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8072 if (fStepping)
8073 rcStrict = VINF_EM_DBG_STEPPED;
8074 }
8075 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8076 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8077 return rcStrict;
8078 }
8079
8080 /*
8081 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8082 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8083 */
8084 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8085 }
8086
8087 /* Validate. */
8088 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8089 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8090 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8091
8092 /* Inject. */
8093 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8094 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8095 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8096 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8097
8098 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8099 && uVector == X86_XCPT_PF)
8100 pMixedCtx->cr2 = GCPtrFaultAddress;
8101
8102 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8103 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8104
8105 AssertRCReturn(rc, rc);
8106 return VINF_SUCCESS;
8107}
8108
8109
8110/**
8111 * Clears the interrupt-window exiting control in the VMCS and if necessary
8112 * clears the current event in the VMCS as well.
8113 *
8114 * @returns VBox status code.
8115 * @param pVCpu The cross context virtual CPU structure.
8116 *
8117 * @remarks Use this function only to clear events that have not yet been
8118 * delivered to the guest but are injected in the VMCS!
8119 * @remarks No-long-jump zone!!!
8120 */
8121static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8122{
8123 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8124
8125 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8126 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8127
8128 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8129 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8130}
8131
8132
8133/**
8134 * Enters the VT-x session.
8135 *
8136 * @returns VBox status code.
8137 * @param pVM The cross context VM structure.
8138 * @param pVCpu The cross context virtual CPU structure.
8139 * @param pCpu Pointer to the CPU info struct.
8140 */
8141VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8142{
8143 AssertPtr(pVM);
8144 AssertPtr(pVCpu);
8145 Assert(pVM->hm.s.vmx.fSupported);
8146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8147 NOREF(pCpu); NOREF(pVM);
8148
8149 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8150 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8151
8152#ifdef VBOX_STRICT
8153 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8154 RTCCUINTREG uHostCR4 = ASMGetCR4();
8155 if (!(uHostCR4 & X86_CR4_VMXE))
8156 {
8157 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8158 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8159 }
8160#endif
8161
8162 /*
8163 * Load the VCPU's VMCS as the current (and active) one.
8164 */
8165 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8166 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8167 if (RT_FAILURE(rc))
8168 return rc;
8169
8170 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8171 pVCpu->hm.s.fLeaveDone = false;
8172 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8173
8174 return VINF_SUCCESS;
8175}
8176
8177
8178/**
8179 * The thread-context callback (only on platforms which support it).
8180 *
8181 * @param enmEvent The thread-context event.
8182 * @param pVCpu The cross context virtual CPU structure.
8183 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8184 * @thread EMT(pVCpu)
8185 */
8186VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8187{
8188 NOREF(fGlobalInit);
8189
8190 switch (enmEvent)
8191 {
8192 case RTTHREADCTXEVENT_OUT:
8193 {
8194 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8195 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8196 VMCPU_ASSERT_EMT(pVCpu);
8197
8198 PVM pVM = pVCpu->CTX_SUFF(pVM);
8199 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8200
8201 /* No longjmps (logger flushes, locks) in this fragile context. */
8202 VMMRZCallRing3Disable(pVCpu);
8203 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8204
8205 /*
8206 * Restore host-state (FPU, debug etc.)
8207 */
8208 if (!pVCpu->hm.s.fLeaveDone)
8209 {
8210 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8211 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8212 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8213 pVCpu->hm.s.fLeaveDone = true;
8214 }
8215
8216 /* Leave HM context, takes care of local init (term). */
8217 int rc = HMR0LeaveCpu(pVCpu);
8218 AssertRC(rc); NOREF(rc);
8219
8220 /* Restore longjmp state. */
8221 VMMRZCallRing3Enable(pVCpu);
8222 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8223 break;
8224 }
8225
8226 case RTTHREADCTXEVENT_IN:
8227 {
8228 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8229 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8230 VMCPU_ASSERT_EMT(pVCpu);
8231
8232 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8233 VMMRZCallRing3Disable(pVCpu);
8234 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8235
8236 /* Initialize the bare minimum state required for HM. This takes care of
8237 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8238 int rc = HMR0EnterCpu(pVCpu);
8239 AssertRC(rc);
8240 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8241
8242 /* Load the active VMCS as the current one. */
8243 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8244 {
8245 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8246 AssertRC(rc); NOREF(rc);
8247 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8248 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8249 }
8250 pVCpu->hm.s.fLeaveDone = false;
8251
8252 /* Restore longjmp state. */
8253 VMMRZCallRing3Enable(pVCpu);
8254 break;
8255 }
8256
8257 default:
8258 break;
8259 }
8260}
8261
8262
8263/**
8264 * Saves the host state in the VMCS host-state.
8265 * Sets up the VM-exit MSR-load area.
8266 *
8267 * The CPU state will be loaded from these fields on every successful VM-exit.
8268 *
8269 * @returns VBox status code.
8270 * @param pVM The cross context VM structure.
8271 * @param pVCpu The cross context virtual CPU structure.
8272 *
8273 * @remarks No-long-jump zone!!!
8274 */
8275static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8276{
8277 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8278
8279 int rc = VINF_SUCCESS;
8280 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8281 {
8282 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8283 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8284
8285 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8286 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8287
8288 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8289 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8290
8291 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8292 }
8293 return rc;
8294}
8295
8296
8297/**
8298 * Saves the host state in the VMCS host-state.
8299 *
8300 * @returns VBox status code.
8301 * @param pVM The cross context VM structure.
8302 * @param pVCpu The cross context virtual CPU structure.
8303 *
8304 * @remarks No-long-jump zone!!!
8305 */
8306VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8307{
8308 AssertPtr(pVM);
8309 AssertPtr(pVCpu);
8310
8311 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8312
8313 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8314 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8315 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8316 return hmR0VmxSaveHostState(pVM, pVCpu);
8317}
8318
8319
8320/**
8321 * Loads the guest state into the VMCS guest-state area.
8322 *
8323 * The will typically be done before VM-entry when the guest-CPU state and the
8324 * VMCS state may potentially be out of sync.
8325 *
8326 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8327 * VM-entry controls.
8328 * Sets up the appropriate VMX non-root function to execute guest code based on
8329 * the guest CPU mode.
8330 *
8331 * @returns VBox strict status code.
8332 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8333 * without unrestricted guest access and the VMMDev is not presently
8334 * mapped (e.g. EFI32).
8335 *
8336 * @param pVM The cross context VM structure.
8337 * @param pVCpu The cross context virtual CPU structure.
8338 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8339 * out-of-sync. Make sure to update the required fields
8340 * before using them.
8341 *
8342 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8343 * caller disables then again on successfull return. Confusing.)
8344 */
8345static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8346{
8347 AssertPtr(pVM);
8348 AssertPtr(pVCpu);
8349 AssertPtr(pMixedCtx);
8350 HMVMX_ASSERT_PREEMPT_SAFE();
8351
8352 VMMRZCallRing3Disable(pVCpu);
8353 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8354
8355 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8356
8357 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8358
8359 /* Determine real-on-v86 mode. */
8360 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8361 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8362 && CPUMIsGuestInRealModeEx(pMixedCtx))
8363 {
8364 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8365 }
8366
8367 /*
8368 * Load the guest-state into the VMCS.
8369 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8370 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8371 */
8372 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8373 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8374
8375 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8376 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8377 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8378
8379 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8380 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8381 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8382
8383 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8384 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8385
8386 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8387 if (rcStrict == VINF_SUCCESS)
8388 { /* likely */ }
8389 else
8390 {
8391 VMMRZCallRing3Enable(pVCpu);
8392 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8393 return rcStrict;
8394 }
8395
8396 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8397 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8398 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8399
8400 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8401 determine we don't have to swap EFER after all. */
8402 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8403 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8404
8405 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8406 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8407
8408 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8409 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8410
8411 /*
8412 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8413 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8414 */
8415 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8416 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8417
8418 /* Clear any unused and reserved bits. */
8419 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8420
8421 VMMRZCallRing3Enable(pVCpu);
8422
8423 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8424 return rc;
8425}
8426
8427
8428/**
8429 * Loads the state shared between the host and guest into the VMCS.
8430 *
8431 * @param pVM The cross context VM structure.
8432 * @param pVCpu The cross context virtual CPU structure.
8433 * @param pCtx Pointer to the guest-CPU context.
8434 *
8435 * @remarks No-long-jump zone!!!
8436 */
8437static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8438{
8439 NOREF(pVM);
8440
8441 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8442 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8443
8444 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8445 {
8446 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8447 AssertRC(rc);
8448 }
8449
8450 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8451 {
8452 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8453 AssertRC(rc);
8454
8455 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8456 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8457 {
8458 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8459 AssertRC(rc);
8460 }
8461 }
8462
8463 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8464 {
8465 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8466 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8467 }
8468
8469 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8470 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8471 {
8472 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8473 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8474 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8475 AssertRC(rc);
8476 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8477 }
8478
8479 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8480 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8481}
8482
8483
8484/**
8485 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8486 *
8487 * @returns Strict VBox status code (i.e. informational status codes too).
8488 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8489 * without unrestricted guest access and the VMMDev is not presently
8490 * mapped (e.g. EFI32).
8491 *
8492 * @param pVM The cross context VM structure.
8493 * @param pVCpu The cross context virtual CPU structure.
8494 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8495 * out-of-sync. Make sure to update the required fields
8496 * before using them.
8497 */
8498static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8499{
8500 HMVMX_ASSERT_PREEMPT_SAFE();
8501
8502 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8503#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8504 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8505#endif
8506
8507 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8508 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8509 {
8510 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8511 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8512 { /* likely */}
8513 else
8514 {
8515 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8516 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8517 }
8518 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8519 }
8520 else if (HMCPU_CF_VALUE(pVCpu))
8521 {
8522 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8523 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8524 { /* likely */}
8525 else
8526 {
8527 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8528 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8529 return rcStrict;
8530 }
8531 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8532 }
8533
8534 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8535 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8536 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8537 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8538 return rcStrict;
8539}
8540
8541
8542/**
8543 * Does the preparations before executing guest code in VT-x.
8544 *
8545 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8546 * recompiler/IEM. We must be cautious what we do here regarding committing
8547 * guest-state information into the VMCS assuming we assuredly execute the
8548 * guest in VT-x mode.
8549 *
8550 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8551 * the common-state (TRPM/forceflags), we must undo those changes so that the
8552 * recompiler/IEM can (and should) use them when it resumes guest execution.
8553 * Otherwise such operations must be done when we can no longer exit to ring-3.
8554 *
8555 * @returns Strict VBox status code (i.e. informational status codes too).
8556 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8557 * have been disabled.
8558 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8559 * double-fault into the guest.
8560 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8561 * dispatched directly.
8562 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8563 *
8564 * @param pVM The cross context VM structure.
8565 * @param pVCpu The cross context virtual CPU structure.
8566 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8567 * out-of-sync. Make sure to update the required fields
8568 * before using them.
8569 * @param pVmxTransient Pointer to the VMX transient structure.
8570 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8571 * us ignore some of the reasons for returning to
8572 * ring-3, and return VINF_EM_DBG_STEPPED if event
8573 * dispatching took place.
8574 */
8575static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8576{
8577 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8578
8579#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8580 PGMRZDynMapFlushAutoSet(pVCpu);
8581#endif
8582
8583 /* Check force flag actions that might require us to go back to ring-3. */
8584 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8585 if (rcStrict == VINF_SUCCESS)
8586 { /* FFs doesn't get set all the time. */ }
8587 else
8588 return rcStrict;
8589
8590#ifndef IEM_VERIFICATION_MODE_FULL
8591 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8592 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8593 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8594 {
8595 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8596 RTGCPHYS GCPhysApicBase;
8597 GCPhysApicBase = pMixedCtx->msrApicBase;
8598 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8599
8600 /* Unalias any existing mapping. */
8601 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8602 AssertRCReturn(rc, rc);
8603
8604 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8605 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8606 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8607 AssertRCReturn(rc, rc);
8608
8609 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8610 }
8611#endif /* !IEM_VERIFICATION_MODE_FULL */
8612
8613 if (TRPMHasTrap(pVCpu))
8614 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8615 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8616
8617 /*
8618 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8619 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8620 */
8621 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8622 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8623 { /* likely */ }
8624 else
8625 {
8626 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8627 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8628 return rcStrict;
8629 }
8630
8631 /*
8632 * Load the guest state bits, we can handle longjmps/getting preempted here.
8633 *
8634 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8635 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8636 * Hence, this needs to be done -after- injection of events.
8637 */
8638 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8639 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8640 { /* likely */ }
8641 else
8642 return rcStrict;
8643
8644 /*
8645 * No longjmps to ring-3 from this point on!!!
8646 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8647 * This also disables flushing of the R0-logger instance (if any).
8648 */
8649 VMMRZCallRing3Disable(pVCpu);
8650
8651 /*
8652 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8653 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8654 *
8655 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8656 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8657 *
8658 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8659 * executing guest code.
8660 */
8661 pVmxTransient->fEFlags = ASMIntDisableFlags();
8662
8663 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8664 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8665 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8666 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8667 {
8668 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8669 {
8670 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8671 pVCpu->hm.s.Event.fPending = false;
8672
8673 return VINF_SUCCESS;
8674 }
8675
8676 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8677 rcStrict = VINF_EM_RAW_INTERRUPT;
8678 }
8679 else
8680 {
8681 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8682 rcStrict = VINF_EM_RAW_TO_R3;
8683 }
8684
8685 ASMSetFlags(pVmxTransient->fEFlags);
8686 VMMRZCallRing3Enable(pVCpu);
8687
8688 return rcStrict;
8689}
8690
8691
8692/**
8693 * Prepares to run guest code in VT-x and we've committed to doing so. This
8694 * means there is no backing out to ring-3 or anywhere else at this
8695 * point.
8696 *
8697 * @param pVM The cross context VM structure.
8698 * @param pVCpu The cross context virtual CPU structure.
8699 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8700 * out-of-sync. Make sure to update the required fields
8701 * before using them.
8702 * @param pVmxTransient Pointer to the VMX transient structure.
8703 *
8704 * @remarks Called with preemption disabled.
8705 * @remarks No-long-jump zone!!!
8706 */
8707static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8708{
8709 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8710 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8711 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8712
8713 /*
8714 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8715 */
8716 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8717 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8718
8719#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8720 if (!CPUMIsGuestFPUStateActive(pVCpu))
8721 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8722 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8723 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8724#endif
8725
8726 if ( pVCpu->hm.s.fPreloadGuestFpu
8727 && !CPUMIsGuestFPUStateActive(pVCpu))
8728 {
8729 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8730 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8731 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8732 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8733 }
8734
8735 /*
8736 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8737 */
8738 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8739 && pVCpu->hm.s.vmx.cMsrs > 0)
8740 {
8741 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8742 }
8743
8744 /*
8745 * Load the host state bits as we may've been preempted (only happens when
8746 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8747 */
8748 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8749 * any effect to the host state needing to be saved? */
8750 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8751 {
8752 /* This ASSUMES that pfnStartVM has been set up already. */
8753 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8754 AssertRC(rc);
8755 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8756 }
8757 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8758
8759 /*
8760 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8761 */
8762 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8763 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8764 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8765
8766 /* Store status of the shared guest-host state at the time of VM-entry. */
8767#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8768 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8769 {
8770 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8771 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8772 }
8773 else
8774#endif
8775 {
8776 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8777 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8778 }
8779 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8780
8781 /*
8782 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8783 */
8784 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8785 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8786
8787 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8788 RTCPUID idCurrentCpu = pCpu->idCpu;
8789 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8790 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8791 {
8792 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8793 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8794 }
8795
8796 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8797 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8798 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8799 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8800
8801 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8802
8803 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8804 to start executing. */
8805
8806 /*
8807 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8808 */
8809 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8810 {
8811 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8812 {
8813 bool fMsrUpdated;
8814 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8815 AssertRC(rc2);
8816 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8817
8818 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8819 &fMsrUpdated);
8820 AssertRC(rc2);
8821 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8822
8823 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8824 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8825 }
8826 else
8827 {
8828 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8829 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8830 }
8831 }
8832
8833#ifdef VBOX_STRICT
8834 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8835 hmR0VmxCheckHostEferMsr(pVCpu);
8836 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8837#endif
8838#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8839 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8840 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8841 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8842#endif
8843}
8844
8845
8846/**
8847 * Performs some essential restoration of state after running guest code in
8848 * VT-x.
8849 *
8850 * @param pVM The cross context VM structure.
8851 * @param pVCpu The cross context virtual CPU structure.
8852 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8853 * out-of-sync. Make sure to update the required fields
8854 * before using them.
8855 * @param pVmxTransient Pointer to the VMX transient structure.
8856 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8857 *
8858 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8859 *
8860 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8861 * unconditionally when it is safe to do so.
8862 */
8863static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8864{
8865 NOREF(pVM);
8866
8867 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8868
8869 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8870 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8871 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8872 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8873 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8874 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8875
8876 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8877 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8878
8879 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8880 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8881 Assert(!ASMIntAreEnabled());
8882 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8883
8884#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8885 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8886 {
8887 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8888 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8889 }
8890#endif
8891
8892#if HC_ARCH_BITS == 64
8893 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8894#endif
8895 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8896#ifdef VBOX_STRICT
8897 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8898#endif
8899 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8900 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8901
8902 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8903 uint32_t uExitReason;
8904 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8905 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8906 AssertRC(rc);
8907 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8908 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8909
8910 /* Update the VM-exit history array. */
8911 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8912
8913 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8914 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8915 {
8916 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8917 pVmxTransient->fVMEntryFailed));
8918 return;
8919 }
8920
8921 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8922 {
8923 /** @todo We can optimize this by only syncing with our force-flags when
8924 * really needed and keeping the VMCS state as it is for most
8925 * VM-exits. */
8926 /* Update the guest interruptibility-state from the VMCS. */
8927 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8928
8929#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8930 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8931 AssertRC(rc);
8932#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8933 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8934 AssertRC(rc);
8935#endif
8936
8937 /*
8938 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8939 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8940 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8941 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8942 */
8943 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8944 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8945 {
8946 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8947 AssertRC(rc);
8948 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8949 }
8950 }
8951}
8952
8953
8954/**
8955 * Runs the guest code using VT-x the normal way.
8956 *
8957 * @returns VBox status code.
8958 * @param pVM The cross context VM structure.
8959 * @param pVCpu The cross context virtual CPU structure.
8960 * @param pCtx Pointer to the guest-CPU context.
8961 *
8962 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8963 */
8964static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8965{
8966 VMXTRANSIENT VmxTransient;
8967 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8968 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8969 uint32_t cLoops = 0;
8970
8971 for (;; cLoops++)
8972 {
8973 Assert(!HMR0SuspendPending());
8974 HMVMX_ASSERT_CPU_SAFE();
8975
8976 /* Preparatory work for running guest code, this may force us to return
8977 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8978 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8979 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8980 if (rcStrict != VINF_SUCCESS)
8981 break;
8982
8983 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8984 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8985 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8986
8987 /* Restore any residual host-state and save any bits shared between host
8988 and guest into the guest-CPU state. Re-enables interrupts! */
8989 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8990
8991 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8992 if (RT_SUCCESS(rcRun))
8993 { /* very likely */ }
8994 else
8995 {
8996 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8997 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8998 return rcRun;
8999 }
9000
9001 /* Profile the VM-exit. */
9002 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9004 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9005 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9006 HMVMX_START_EXIT_DISPATCH_PROF();
9007
9008 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9009
9010 /* Handle the VM-exit. */
9011#ifdef HMVMX_USE_FUNCTION_TABLE
9012 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9013#else
9014 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9015#endif
9016 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9017 if (rcStrict == VINF_SUCCESS)
9018 {
9019 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9020 continue; /* likely */
9021 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9022 rcStrict = VINF_EM_RAW_INTERRUPT;
9023 }
9024 break;
9025 }
9026
9027 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9028 return rcStrict;
9029}
9030
9031
9032
9033/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9034 * probes.
9035 *
9036 * The following few functions and associated structure contains the bloat
9037 * necessary for providing detailed debug events and dtrace probes as well as
9038 * reliable host side single stepping. This works on the principle of
9039 * "subclassing" the normal execution loop and workers. We replace the loop
9040 * method completely and override selected helpers to add necessary adjustments
9041 * to their core operation.
9042 *
9043 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9044 * any performance for debug and analysis features.
9045 *
9046 * @{
9047 */
9048
9049typedef struct VMXRUNDBGSTATE
9050{
9051 /** The RIP we started executing at. This is for detecting that we stepped. */
9052 uint64_t uRipStart;
9053 /** The CS we started executing with. */
9054 uint16_t uCsStart;
9055
9056 /** Whether we've actually modified the 1st execution control field. */
9057 bool fModifiedProcCtls : 1;
9058 /** Whether we've actually modified the 2nd execution control field. */
9059 bool fModifiedProcCtls2 : 1;
9060 /** Whether we've actually modified the exception bitmap. */
9061 bool fModifiedXcptBitmap : 1;
9062
9063 /** We desire the modified the CR0 mask to be cleared. */
9064 bool fClearCr0Mask : 1;
9065 /** We desire the modified the CR4 mask to be cleared. */
9066 bool fClearCr4Mask : 1;
9067 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9068 uint32_t fCpe1Extra;
9069 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9070 uint32_t fCpe1Unwanted;
9071 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9072 uint32_t fCpe2Extra;
9073 /** Extra stuff we need in */
9074 uint32_t bmXcptExtra;
9075 /** The sequence number of the Dtrace provider settings the state was
9076 * configured against. */
9077 uint32_t uDtraceSettingsSeqNo;
9078 /** Exits to check (one bit per exit). */
9079 uint32_t bmExitsToCheck[3];
9080
9081 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9082 uint32_t fProcCtlsInitial;
9083 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9084 uint32_t fProcCtls2Initial;
9085 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9086 uint32_t bmXcptInitial;
9087} VMXRUNDBGSTATE;
9088AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9089typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9090
9091
9092/**
9093 * Initializes the VMXRUNDBGSTATE structure.
9094 *
9095 * @param pVCpu The cross context virtual CPU structure of the
9096 * calling EMT.
9097 * @param pCtx The CPU register context to go with @a pVCpu.
9098 * @param pDbgState The structure to initialize.
9099 */
9100DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9101{
9102 pDbgState->uRipStart = pCtx->rip;
9103 pDbgState->uCsStart = pCtx->cs.Sel;
9104
9105 pDbgState->fModifiedProcCtls = false;
9106 pDbgState->fModifiedProcCtls2 = false;
9107 pDbgState->fModifiedXcptBitmap = false;
9108 pDbgState->fClearCr0Mask = false;
9109 pDbgState->fClearCr4Mask = false;
9110 pDbgState->fCpe1Extra = 0;
9111 pDbgState->fCpe1Unwanted = 0;
9112 pDbgState->fCpe2Extra = 0;
9113 pDbgState->bmXcptExtra = 0;
9114 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9115 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9116 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9117}
9118
9119
9120/**
9121 * Updates the VMSC fields with changes requested by @a pDbgState.
9122 *
9123 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9124 * immediately before executing guest code, i.e. when interrupts are disabled.
9125 * We don't check status codes here as we cannot easily assert or return in the
9126 * latter case.
9127 *
9128 * @param pVCpu The cross context virtual CPU structure.
9129 * @param pDbgState The debug state.
9130 */
9131DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9132{
9133 /*
9134 * Ensure desired flags in VMCS control fields are set.
9135 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9136 *
9137 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9138 * there should be no stale data in pCtx at this point.
9139 */
9140 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9141 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9142 {
9143 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9144 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9145 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9146 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9147 pDbgState->fModifiedProcCtls = true;
9148 }
9149
9150 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9151 {
9152 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9153 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9154 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9155 pDbgState->fModifiedProcCtls2 = true;
9156 }
9157
9158 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9159 {
9160 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9161 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9162 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9163 pDbgState->fModifiedXcptBitmap = true;
9164 }
9165
9166 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9167 {
9168 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9169 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9170 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9171 }
9172
9173 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9174 {
9175 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9176 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9177 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9178 }
9179}
9180
9181
9182DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9183{
9184 /*
9185 * Restore exit control settings as we may not reenter this function the
9186 * next time around.
9187 */
9188 /* We reload the initial value, trigger what we can of recalculations the
9189 next time around. From the looks of things, that's all that's required atm. */
9190 if (pDbgState->fModifiedProcCtls)
9191 {
9192 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9193 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9194 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9195 AssertRCReturn(rc2, rc2);
9196 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9197 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9198 }
9199
9200 /* We're currently the only ones messing with this one, so just restore the
9201 cached value and reload the field. */
9202 if ( pDbgState->fModifiedProcCtls2
9203 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9204 {
9205 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9206 AssertRCReturn(rc2, rc2);
9207 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9208 }
9209
9210 /* If we've modified the exception bitmap, we restore it and trigger
9211 reloading and partial recalculation the next time around. */
9212 if (pDbgState->fModifiedXcptBitmap)
9213 {
9214 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9215 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9216 }
9217
9218 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9219 if (pDbgState->fClearCr0Mask)
9220 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9221
9222 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9223 if (pDbgState->fClearCr4Mask)
9224 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9225
9226 return rcStrict;
9227}
9228
9229
9230/**
9231 * Configures VM-exit controls for current DBGF and DTrace settings.
9232 *
9233 * This updates @a pDbgState and the VMCS execution control fields to reflect
9234 * the necessary exits demanded by DBGF and DTrace.
9235 *
9236 * @param pVM The cross context VM structure.
9237 * @param pVCpu The cross context virtual CPU structure.
9238 * @param pCtx Pointer to the guest-CPU context.
9239 * @param pDbgState The debug state.
9240 * @param pVmxTransient Pointer to the VMX transient structure. May update
9241 * fUpdateTscOffsettingAndPreemptTimer.
9242 */
9243static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9244 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9245{
9246 /*
9247 * Take down the dtrace serial number so we can spot changes.
9248 */
9249 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9250 ASMCompilerBarrier();
9251
9252 /*
9253 * We'll rebuild most of the middle block of data members (holding the
9254 * current settings) as we go along here, so start by clearing it all.
9255 */
9256 pDbgState->bmXcptExtra = 0;
9257 pDbgState->fCpe1Extra = 0;
9258 pDbgState->fCpe1Unwanted = 0;
9259 pDbgState->fCpe2Extra = 0;
9260 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9261 pDbgState->bmExitsToCheck[i] = 0;
9262
9263 /*
9264 * Software interrupts (INT XXh) - no idea how to trigger these...
9265 */
9266 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9267 || VBOXVMM_INT_SOFTWARE_ENABLED())
9268 {
9269 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9270 }
9271
9272 /*
9273 * Exception bitmap and XCPT events+probes.
9274 */
9275 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9276 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9277 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9278
9279 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9280 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9281 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9282 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9283 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9284 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9285 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9286 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9287 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9288 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9289 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9290 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9291 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9292 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9293 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9294 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9295 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9296 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9297
9298 if (pDbgState->bmXcptExtra)
9299 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9300
9301 /*
9302 * Process events and probes for VM exits, making sure we get the wanted exits.
9303 *
9304 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9305 * So, when adding/changing/removing please don't forget to update it.
9306 *
9307 * Some of the macros are picking up local variables to save horizontal space,
9308 * (being able to see it in a table is the lesser evil here).
9309 */
9310#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9311 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9312 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9313#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9314 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9315 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9316 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9317 } else do { } while (0)
9318#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9319 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9320 { \
9321 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9322 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9323 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9324 } else do { } while (0)
9325#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9326 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9327 { \
9328 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9329 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9330 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9331 } else do { } while (0)
9332#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9333 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9334 { \
9335 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9336 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9337 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9338 } else do { } while (0)
9339
9340 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9341 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9342 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9343 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9344 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9345
9346 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9347 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9348 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9349 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9350 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9351 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9352 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9353 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9354 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9355 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9356 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9357 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9358 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9359 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9360 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9361 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9362 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9363 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9364 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9365 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9366 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9367 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9368 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9369 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9370 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9371 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9372 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9373 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9374 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9375 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9376 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9377 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9378 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9379 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9380 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9381 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9382
9383 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9384 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9385 {
9386 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9387 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9388 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9389 AssertRC(rc2);
9390
9391#if 0 /** @todo fix me */
9392 pDbgState->fClearCr0Mask = true;
9393 pDbgState->fClearCr4Mask = true;
9394#endif
9395 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9396 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9397 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9398 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9399 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9400 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9401 require clearing here and in the loop if we start using it. */
9402 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9403 }
9404 else
9405 {
9406 if (pDbgState->fClearCr0Mask)
9407 {
9408 pDbgState->fClearCr0Mask = false;
9409 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9410 }
9411 if (pDbgState->fClearCr4Mask)
9412 {
9413 pDbgState->fClearCr4Mask = false;
9414 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9415 }
9416 }
9417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9419
9420 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9421 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9422 {
9423 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9424 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9425 }
9426 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9427 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9428
9429 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9431 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9432 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9433 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9435 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9436 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9437#if 0 /** @todo too slow, fix handler. */
9438 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9439#endif
9440 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9441
9442 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9443 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9444 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9445 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9446 {
9447 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9448 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9449 }
9450 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9451 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9452 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9453 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9454
9455 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9456 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9457 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9458 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9459 {
9460 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9461 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9462 }
9463 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9464 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9465 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9466 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9467
9468 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9469 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9470 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9471 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9472 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9473 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9474 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9475 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9476 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9477 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9478 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9479 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9480 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9481 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9482 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9483 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9484 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9485 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9486 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9487 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9488 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9489 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9490
9491#undef IS_EITHER_ENABLED
9492#undef SET_ONLY_XBM_IF_EITHER_EN
9493#undef SET_CPE1_XBM_IF_EITHER_EN
9494#undef SET_CPEU_XBM_IF_EITHER_EN
9495#undef SET_CPE2_XBM_IF_EITHER_EN
9496
9497 /*
9498 * Sanitize the control stuff.
9499 */
9500 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9501 if (pDbgState->fCpe2Extra)
9502 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9503 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9504 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9505 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9506 {
9507 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9508 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9509 }
9510
9511 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9512 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9513 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9514 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9515}
9516
9517
9518/**
9519 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9520 *
9521 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9522 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9523 * either.
9524 *
9525 * @returns Strict VBox status code (i.e. informational status codes too).
9526 * @param pVM The cross context VM structure.
9527 * @param pVCpu The cross context virtual CPU structure.
9528 * @param pMixedCtx Pointer to the guest-CPU context.
9529 * @param pVmxTransient Pointer to the VMX-transient structure.
9530 * @param uExitReason The VM-exit reason.
9531 *
9532 * @remarks The name of this function is displayed by dtrace, so keep it short
9533 * and to the point. No longer than 33 chars long, please.
9534 */
9535static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9536 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9537{
9538 /*
9539 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9540 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9541 *
9542 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9543 * does. Must add/change/remove both places. Same ordering, please.
9544 *
9545 * Added/removed events must also be reflected in the next section
9546 * where we dispatch dtrace events.
9547 */
9548 bool fDtrace1 = false;
9549 bool fDtrace2 = false;
9550 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9551 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9552 uint32_t uEventArg = 0;
9553#define SET_EXIT(a_EventSubName) \
9554 do { \
9555 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9556 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9557 } while (0)
9558#define SET_BOTH(a_EventSubName) \
9559 do { \
9560 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9561 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9562 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9563 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9564 } while (0)
9565 switch (uExitReason)
9566 {
9567 case VMX_EXIT_MTF:
9568 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9569
9570 case VMX_EXIT_XCPT_OR_NMI:
9571 {
9572 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9573 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9574 {
9575 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9576 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9577 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9578 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9579 {
9580 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9581 {
9582 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9583 uEventArg = pVmxTransient->uExitIntErrorCode;
9584 }
9585 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9586 switch (enmEvent1)
9587 {
9588 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9589 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9590 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9591 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9592 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9593 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9594 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9595 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9596 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9597 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9598 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9599 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9600 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9601 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9602 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9603 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9604 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9605 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9606 default: break;
9607 }
9608 }
9609 else
9610 AssertFailed();
9611 break;
9612
9613 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9614 uEventArg = idxVector;
9615 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9616 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9617 break;
9618 }
9619 break;
9620 }
9621
9622 case VMX_EXIT_TRIPLE_FAULT:
9623 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9624 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9625 break;
9626 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9627 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9628 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9629 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9630 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9631
9632 /* Instruction specific VM-exits: */
9633 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9634 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9635 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9636 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9637 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9638 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9639 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9640 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9641 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9642 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9643 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9644 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9645 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9646 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9647 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9648 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9649 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9650 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9651 case VMX_EXIT_MOV_CRX:
9652 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9653/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9654* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9655 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9656 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9657 SET_BOTH(CRX_READ);
9658 else
9659 SET_BOTH(CRX_WRITE);
9660 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9661 break;
9662 case VMX_EXIT_MOV_DRX:
9663 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9664 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9665 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9666 SET_BOTH(DRX_READ);
9667 else
9668 SET_BOTH(DRX_WRITE);
9669 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9670 break;
9671 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9672 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9673 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9674 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9675 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9676 case VMX_EXIT_XDTR_ACCESS:
9677 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9678 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9679 {
9680 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9681 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9682 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9683 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9684 }
9685 break;
9686
9687 case VMX_EXIT_TR_ACCESS:
9688 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9689 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9690 {
9691 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9692 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9693 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9694 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9695 }
9696 break;
9697
9698 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9699 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9700 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9701 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9702 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9703 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9704 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9705 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9706 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9707 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9708 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9709
9710 /* Events that aren't relevant at this point. */
9711 case VMX_EXIT_EXT_INT:
9712 case VMX_EXIT_INT_WINDOW:
9713 case VMX_EXIT_NMI_WINDOW:
9714 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9715 case VMX_EXIT_PREEMPT_TIMER:
9716 case VMX_EXIT_IO_INSTR:
9717 break;
9718
9719 /* Errors and unexpected events. */
9720 case VMX_EXIT_INIT_SIGNAL:
9721 case VMX_EXIT_SIPI:
9722 case VMX_EXIT_IO_SMI:
9723 case VMX_EXIT_SMI:
9724 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9725 case VMX_EXIT_ERR_MSR_LOAD:
9726 case VMX_EXIT_ERR_MACHINE_CHECK:
9727 break;
9728
9729 default:
9730 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9731 break;
9732 }
9733#undef SET_BOTH
9734#undef SET_EXIT
9735
9736 /*
9737 * Dtrace tracepoints go first. We do them here at once so we don't
9738 * have to copy the guest state saving and stuff a few dozen times.
9739 * Down side is that we've got to repeat the switch, though this time
9740 * we use enmEvent since the probes are a subset of what DBGF does.
9741 */
9742 if (fDtrace1 || fDtrace2)
9743 {
9744 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9745 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9746 switch (enmEvent1)
9747 {
9748 /** @todo consider which extra parameters would be helpful for each probe. */
9749 case DBGFEVENT_END: break;
9750 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9751 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9752 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9753 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9754 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9755 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9756 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9757 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9758 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9759 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9760 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9761 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9762 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9763 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9764 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9765 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9766 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9767 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9768 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9769 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9770 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9771 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9772 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9773 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9774 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9775 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9776 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9777 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9778 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9779 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9780 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9781 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9782 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9783 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9784 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9785 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9786 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9787 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9788 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9789 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9790 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9791 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9792 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9793 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9794 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9795 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9796 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9797 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9798 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9799 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9800 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9801 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9802 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9803 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9804 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9805 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9806 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9807 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9808 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9809 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9810 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9811 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9812 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9813 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9814 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9815 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9816 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9817 }
9818 switch (enmEvent2)
9819 {
9820 /** @todo consider which extra parameters would be helpful for each probe. */
9821 case DBGFEVENT_END: break;
9822 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9823 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9824 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9825 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9826 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9827 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9828 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9829 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9830 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9831 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9832 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9833 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9834 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9835 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9836 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9837 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9838 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9839 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9840 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9841 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9842 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9843 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9844 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9845 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9846 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9847 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9848 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9849 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9850 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9852 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9853 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9854 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9855 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9856 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9857 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9858 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9866 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9867 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9868 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9871 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9872 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9874 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9875 }
9876 }
9877
9878 /*
9879 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9880 * the DBGF call will do a full check).
9881 *
9882 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9883 * Note! If we have to events, we prioritize the first, i.e. the instruction
9884 * one, in order to avoid event nesting.
9885 */
9886 if ( enmEvent1 != DBGFEVENT_END
9887 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9888 {
9889 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9890 if (rcStrict != VINF_SUCCESS)
9891 return rcStrict;
9892 }
9893 else if ( enmEvent2 != DBGFEVENT_END
9894 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9895 {
9896 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9897 if (rcStrict != VINF_SUCCESS)
9898 return rcStrict;
9899 }
9900
9901 return VINF_SUCCESS;
9902}
9903
9904
9905/**
9906 * Single-stepping VM-exit filtering.
9907 *
9908 * This is preprocessing the exits and deciding whether we've gotten far enough
9909 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9910 * performed.
9911 *
9912 * @returns Strict VBox status code (i.e. informational status codes too).
9913 * @param pVM The cross context VM structure.
9914 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9915 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9916 * out-of-sync. Make sure to update the required
9917 * fields before using them.
9918 * @param pVmxTransient Pointer to the VMX-transient structure.
9919 * @param uExitReason The VM-exit reason.
9920 * @param pDbgState The debug state.
9921 */
9922DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9923 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9924{
9925 /*
9926 * Expensive (saves context) generic dtrace exit probe.
9927 */
9928 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9929 { /* more likely */ }
9930 else
9931 {
9932 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9933 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9934 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9935 }
9936
9937 /*
9938 * Check for host NMI, just to get that out of the way.
9939 */
9940 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9941 { /* normally likely */ }
9942 else
9943 {
9944 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9945 AssertRCReturn(rc2, rc2);
9946 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9947 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9948 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9949 }
9950
9951 /*
9952 * Check for single stepping event if we're stepping.
9953 */
9954 if (pVCpu->hm.s.fSingleInstruction)
9955 {
9956 switch (uExitReason)
9957 {
9958 case VMX_EXIT_MTF:
9959 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9960
9961 /* Various events: */
9962 case VMX_EXIT_XCPT_OR_NMI:
9963 case VMX_EXIT_EXT_INT:
9964 case VMX_EXIT_TRIPLE_FAULT:
9965 case VMX_EXIT_INT_WINDOW:
9966 case VMX_EXIT_NMI_WINDOW:
9967 case VMX_EXIT_TASK_SWITCH:
9968 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9969 case VMX_EXIT_APIC_ACCESS:
9970 case VMX_EXIT_EPT_VIOLATION:
9971 case VMX_EXIT_EPT_MISCONFIG:
9972 case VMX_EXIT_PREEMPT_TIMER:
9973
9974 /* Instruction specific VM-exits: */
9975 case VMX_EXIT_CPUID:
9976 case VMX_EXIT_GETSEC:
9977 case VMX_EXIT_HLT:
9978 case VMX_EXIT_INVD:
9979 case VMX_EXIT_INVLPG:
9980 case VMX_EXIT_RDPMC:
9981 case VMX_EXIT_RDTSC:
9982 case VMX_EXIT_RSM:
9983 case VMX_EXIT_VMCALL:
9984 case VMX_EXIT_VMCLEAR:
9985 case VMX_EXIT_VMLAUNCH:
9986 case VMX_EXIT_VMPTRLD:
9987 case VMX_EXIT_VMPTRST:
9988 case VMX_EXIT_VMREAD:
9989 case VMX_EXIT_VMRESUME:
9990 case VMX_EXIT_VMWRITE:
9991 case VMX_EXIT_VMXOFF:
9992 case VMX_EXIT_VMXON:
9993 case VMX_EXIT_MOV_CRX:
9994 case VMX_EXIT_MOV_DRX:
9995 case VMX_EXIT_IO_INSTR:
9996 case VMX_EXIT_RDMSR:
9997 case VMX_EXIT_WRMSR:
9998 case VMX_EXIT_MWAIT:
9999 case VMX_EXIT_MONITOR:
10000 case VMX_EXIT_PAUSE:
10001 case VMX_EXIT_XDTR_ACCESS:
10002 case VMX_EXIT_TR_ACCESS:
10003 case VMX_EXIT_INVEPT:
10004 case VMX_EXIT_RDTSCP:
10005 case VMX_EXIT_INVVPID:
10006 case VMX_EXIT_WBINVD:
10007 case VMX_EXIT_XSETBV:
10008 case VMX_EXIT_RDRAND:
10009 case VMX_EXIT_INVPCID:
10010 case VMX_EXIT_VMFUNC:
10011 case VMX_EXIT_RDSEED:
10012 case VMX_EXIT_XSAVES:
10013 case VMX_EXIT_XRSTORS:
10014 {
10015 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10016 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10017 AssertRCReturn(rc2, rc2);
10018 if ( pMixedCtx->rip != pDbgState->uRipStart
10019 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10020 return VINF_EM_DBG_STEPPED;
10021 break;
10022 }
10023
10024 /* Errors and unexpected events: */
10025 case VMX_EXIT_INIT_SIGNAL:
10026 case VMX_EXIT_SIPI:
10027 case VMX_EXIT_IO_SMI:
10028 case VMX_EXIT_SMI:
10029 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10030 case VMX_EXIT_ERR_MSR_LOAD:
10031 case VMX_EXIT_ERR_MACHINE_CHECK:
10032 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10033 break;
10034
10035 default:
10036 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
10037 break;
10038 }
10039 }
10040
10041 /*
10042 * Check for debugger event breakpoints and dtrace probes.
10043 */
10044 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10045 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10046 {
10047 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10048 if (rcStrict != VINF_SUCCESS)
10049 return rcStrict;
10050 }
10051
10052 /*
10053 * Normal processing.
10054 */
10055#ifdef HMVMX_USE_FUNCTION_TABLE
10056 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10057#else
10058 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10059#endif
10060}
10061
10062
10063/**
10064 * Single steps guest code using VT-x.
10065 *
10066 * @returns Strict VBox status code (i.e. informational status codes too).
10067 * @param pVM The cross context VM structure.
10068 * @param pVCpu The cross context virtual CPU structure.
10069 * @param pCtx Pointer to the guest-CPU context.
10070 *
10071 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10072 */
10073static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10074{
10075 VMXTRANSIENT VmxTransient;
10076 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10077
10078 /* Set HMCPU indicators. */
10079 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10080 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10081 pVCpu->hm.s.fDebugWantRdTscExit = false;
10082 pVCpu->hm.s.fUsingDebugLoop = true;
10083
10084 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10085 VMXRUNDBGSTATE DbgState;
10086 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10087 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10088
10089 /*
10090 * The loop.
10091 */
10092 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10093 for (uint32_t cLoops = 0; ; cLoops++)
10094 {
10095 Assert(!HMR0SuspendPending());
10096 HMVMX_ASSERT_CPU_SAFE();
10097 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10098
10099 /*
10100 * Preparatory work for running guest code, this may force us to return
10101 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10102 */
10103 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10104 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10105 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10106 if (rcStrict != VINF_SUCCESS)
10107 break;
10108
10109 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10110 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10111
10112 /*
10113 * Now we can run the guest code.
10114 */
10115 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10116
10117 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10118
10119 /*
10120 * Restore any residual host-state and save any bits shared between host
10121 * and guest into the guest-CPU state. Re-enables interrupts!
10122 */
10123 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10124
10125 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10126 if (RT_SUCCESS(rcRun))
10127 { /* very likely */ }
10128 else
10129 {
10130 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10131 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10132 return rcRun;
10133 }
10134
10135 /* Profile the VM-exit. */
10136 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10137 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10138 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10139 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10140 HMVMX_START_EXIT_DISPATCH_PROF();
10141
10142 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10143
10144 /*
10145 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10146 */
10147 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10148 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10149 if (rcStrict != VINF_SUCCESS)
10150 break;
10151 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10152 {
10153 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10154 rcStrict = VINF_EM_RAW_INTERRUPT;
10155 break;
10156 }
10157
10158 /*
10159 * Stepping: Did the RIP change, if so, consider it a single step.
10160 * Otherwise, make sure one of the TFs gets set.
10161 */
10162 if (fStepping)
10163 {
10164 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10165 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10166 AssertRCReturn(rc2, rc2);
10167 if ( pCtx->rip != DbgState.uRipStart
10168 || pCtx->cs.Sel != DbgState.uCsStart)
10169 {
10170 rcStrict = VINF_EM_DBG_STEPPED;
10171 break;
10172 }
10173 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10174 }
10175
10176 /*
10177 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10178 */
10179 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10180 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10181 }
10182
10183 /*
10184 * Clear the X86_EFL_TF if necessary.
10185 */
10186 if (pVCpu->hm.s.fClearTrapFlag)
10187 {
10188 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10189 AssertRCReturn(rc2, rc2);
10190 pVCpu->hm.s.fClearTrapFlag = false;
10191 pCtx->eflags.Bits.u1TF = 0;
10192 }
10193 /** @todo there seems to be issues with the resume flag when the monitor trap
10194 * flag is pending without being used. Seen early in bios init when
10195 * accessing APIC page in protected mode. */
10196
10197 /*
10198 * Restore VM-exit control settings as we may not reenter this function the
10199 * next time around.
10200 */
10201 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10202
10203 /* Restore HMCPU indicators. */
10204 pVCpu->hm.s.fUsingDebugLoop = false;
10205 pVCpu->hm.s.fDebugWantRdTscExit = false;
10206 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10207
10208 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10209 return rcStrict;
10210}
10211
10212
10213/** @} */
10214
10215
10216/**
10217 * Checks if any expensive dtrace probes are enabled and we should go to the
10218 * debug loop.
10219 *
10220 * @returns true if we should use debug loop, false if not.
10221 */
10222static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10223{
10224 /* It's probably faster to OR the raw 32-bit counter variables together.
10225 Since the variables are in an array and the probes are next to one
10226 another (more or less), we have good locality. So, better read
10227 eight-nine cache lines ever time and only have one conditional, than
10228 128+ conditionals, right? */
10229 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10230 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10231 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10232 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10233 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10234 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10235 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10236 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10237 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10238 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10239 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10240 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10241 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10242 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10243 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10244 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10245 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10246 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10247 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10248 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10249 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10250 ) != 0
10251 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10252 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10253 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10254 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10255 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10256 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10257 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10258 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10259 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10260 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10261 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10262 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10263 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10264 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10265 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10266 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10267 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10268 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10269 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10270 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10271 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10272 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10273 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10274 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10275 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10276 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10277 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10278 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10279 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10280 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10281 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10282 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10283 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10284 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10285 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10286 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10287 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10288 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10289 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10290 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10291 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10292 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10293 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10294 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10295 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10296 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10297 ) != 0
10298 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10299 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10300 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10301 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10302 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10303 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10304 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10305 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10306 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10307 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10308 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10309 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10310 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10311 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10312 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10313 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10314 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10315 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10316 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10317 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10318 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10319 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10320 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10321 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10322 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10323 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10324 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10325 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10326 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10327 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10328 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10329 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10330 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10331 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10332 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10333 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10334 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10335 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10336 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10337 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10338 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10339 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10340 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10341 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10342 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10343 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10344 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10345 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10346 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10347 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10348 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10349 ) != 0;
10350}
10351
10352
10353/**
10354 * Runs the guest code using VT-x.
10355 *
10356 * @returns Strict VBox status code (i.e. informational status codes too).
10357 * @param pVM The cross context VM structure.
10358 * @param pVCpu The cross context virtual CPU structure.
10359 * @param pCtx Pointer to the guest-CPU context.
10360 */
10361VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10362{
10363 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10364 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10365 HMVMX_ASSERT_PREEMPT_SAFE();
10366
10367 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10368
10369 VBOXSTRICTRC rcStrict;
10370 if ( !pVCpu->hm.s.fUseDebugLoop
10371 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10372 && !DBGFIsStepping(pVCpu) )
10373 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10374 else
10375 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10376
10377 if (rcStrict == VERR_EM_INTERPRETER)
10378 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10379 else if (rcStrict == VINF_EM_RESET)
10380 rcStrict = VINF_EM_TRIPLE_FAULT;
10381
10382 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10383 if (RT_FAILURE(rc2))
10384 {
10385 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10386 rcStrict = rc2;
10387 }
10388 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10389 return rcStrict;
10390}
10391
10392
10393#ifndef HMVMX_USE_FUNCTION_TABLE
10394DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10395{
10396# ifdef DEBUG_ramshankar
10397# define RETURN_EXIT_CALL(a_CallExpr) \
10398 do { \
10399 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10400 VBOXSTRICTRC rcStrict = a_CallExpr; \
10401 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10402 return rcStrict; \
10403 } while (0)
10404# else
10405# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10406# endif
10407 switch (rcReason)
10408 {
10409 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10410 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10411 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10412 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10413 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10414 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10415 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10416 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10417 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10418 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10419 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10420 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10421 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10422 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10423 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10424 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10425 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10426 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10427 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10428 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10429 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10430 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10431 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10432 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10433 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10434 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10435 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10436 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10437 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10438 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10439 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10440 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10441 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10442 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10443
10444 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10445 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10446 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10447 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10448 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10449 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10450 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10451 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10452 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10453
10454 case VMX_EXIT_VMCLEAR:
10455 case VMX_EXIT_VMLAUNCH:
10456 case VMX_EXIT_VMPTRLD:
10457 case VMX_EXIT_VMPTRST:
10458 case VMX_EXIT_VMREAD:
10459 case VMX_EXIT_VMRESUME:
10460 case VMX_EXIT_VMWRITE:
10461 case VMX_EXIT_VMXOFF:
10462 case VMX_EXIT_VMXON:
10463 case VMX_EXIT_INVEPT:
10464 case VMX_EXIT_INVVPID:
10465 case VMX_EXIT_VMFUNC:
10466 case VMX_EXIT_XSAVES:
10467 case VMX_EXIT_XRSTORS:
10468 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10469 case VMX_EXIT_RESERVED_60:
10470 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10471 case VMX_EXIT_RESERVED_62:
10472 default:
10473 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10474 }
10475#undef RETURN_EXIT_CALL
10476}
10477#endif /* !HMVMX_USE_FUNCTION_TABLE */
10478
10479
10480#ifdef VBOX_STRICT
10481/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10482# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10483 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10484
10485# define HMVMX_ASSERT_PREEMPT_CPUID() \
10486 do { \
10487 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10488 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10489 } while (0)
10490
10491# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10492 do { \
10493 AssertPtr(pVCpu); \
10494 AssertPtr(pMixedCtx); \
10495 AssertPtr(pVmxTransient); \
10496 Assert(pVmxTransient->fVMEntryFailed == false); \
10497 Assert(ASMIntAreEnabled()); \
10498 HMVMX_ASSERT_PREEMPT_SAFE(); \
10499 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10500 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10501 HMVMX_ASSERT_PREEMPT_SAFE(); \
10502 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10503 HMVMX_ASSERT_PREEMPT_CPUID(); \
10504 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10505 } while (0)
10506
10507# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10508 do { \
10509 Log4Func(("\n")); \
10510 } while (0)
10511#else /* nonstrict builds: */
10512# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10513 do { \
10514 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10515 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10516 } while (0)
10517# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10518#endif
10519
10520
10521/**
10522 * Advances the guest RIP by the specified number of bytes.
10523 *
10524 * @param pVCpu The cross context virtual CPU structure.
10525 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10526 * out-of-sync. Make sure to update the required fields
10527 * before using them.
10528 * @param cbInstr Number of bytes to advance the RIP by.
10529 *
10530 * @remarks No-long-jump zone!!!
10531 */
10532DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10533{
10534 /* Advance the RIP. */
10535 pMixedCtx->rip += cbInstr;
10536 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10537
10538 /* Update interrupt inhibition. */
10539 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10540 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10541 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10542}
10543
10544
10545/**
10546 * Advances the guest RIP after reading it from the VMCS.
10547 *
10548 * @returns VBox status code, no informational status codes.
10549 * @param pVCpu The cross context virtual CPU structure.
10550 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10551 * out-of-sync. Make sure to update the required fields
10552 * before using them.
10553 * @param pVmxTransient Pointer to the VMX transient structure.
10554 *
10555 * @remarks No-long-jump zone!!!
10556 */
10557static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10558{
10559 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10560 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10561 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10562 AssertRCReturn(rc, rc);
10563
10564 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10565
10566 /*
10567 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10568 * pending debug exception field as it takes care of priority of events.
10569 *
10570 * See Intel spec. 32.2.1 "Debug Exceptions".
10571 */
10572 if ( !pVCpu->hm.s.fSingleInstruction
10573 && pMixedCtx->eflags.Bits.u1TF)
10574 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10575
10576 return VINF_SUCCESS;
10577}
10578
10579
10580/**
10581 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10582 * and update error record fields accordingly.
10583 *
10584 * @return VMX_IGS_* return codes.
10585 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10586 * wrong with the guest state.
10587 *
10588 * @param pVM The cross context VM structure.
10589 * @param pVCpu The cross context virtual CPU structure.
10590 * @param pCtx Pointer to the guest-CPU state.
10591 *
10592 * @remarks This function assumes our cache of the VMCS controls
10593 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10594 */
10595static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10596{
10597#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10598#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10599 uError = (err); \
10600 break; \
10601 } else do { } while (0)
10602
10603 int rc;
10604 uint32_t uError = VMX_IGS_ERROR;
10605 uint32_t u32Val;
10606 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10607
10608 do
10609 {
10610 /*
10611 * CR0.
10612 */
10613 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10614 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10615 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10616 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10617 if (fUnrestrictedGuest)
10618 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10619
10620 uint32_t u32GuestCR0;
10621 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10622 AssertRCBreak(rc);
10623 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10624 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10625 if ( !fUnrestrictedGuest
10626 && (u32GuestCR0 & X86_CR0_PG)
10627 && !(u32GuestCR0 & X86_CR0_PE))
10628 {
10629 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10630 }
10631
10632 /*
10633 * CR4.
10634 */
10635 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10636 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10637
10638 uint32_t u32GuestCR4;
10639 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10640 AssertRCBreak(rc);
10641 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10642 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10643
10644 /*
10645 * IA32_DEBUGCTL MSR.
10646 */
10647 uint64_t u64Val;
10648 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10649 AssertRCBreak(rc);
10650 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10651 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10652 {
10653 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10654 }
10655 uint64_t u64DebugCtlMsr = u64Val;
10656
10657#ifdef VBOX_STRICT
10658 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10659 AssertRCBreak(rc);
10660 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10661#endif
10662 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10663
10664 /*
10665 * RIP and RFLAGS.
10666 */
10667 uint32_t u32Eflags;
10668#if HC_ARCH_BITS == 64
10669 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10670 AssertRCBreak(rc);
10671 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10672 if ( !fLongModeGuest
10673 || !pCtx->cs.Attr.n.u1Long)
10674 {
10675 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10676 }
10677 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10678 * must be identical if the "IA-32e mode guest" VM-entry
10679 * control is 1 and CS.L is 1. No check applies if the
10680 * CPU supports 64 linear-address bits. */
10681
10682 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10683 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10684 AssertRCBreak(rc);
10685 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10686 VMX_IGS_RFLAGS_RESERVED);
10687 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10688 u32Eflags = u64Val;
10689#else
10690 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10691 AssertRCBreak(rc);
10692 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10693 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10694#endif
10695
10696 if ( fLongModeGuest
10697 || ( fUnrestrictedGuest
10698 && !(u32GuestCR0 & X86_CR0_PE)))
10699 {
10700 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10701 }
10702
10703 uint32_t u32EntryInfo;
10704 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10705 AssertRCBreak(rc);
10706 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10707 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10708 {
10709 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10710 }
10711
10712 /*
10713 * 64-bit checks.
10714 */
10715#if HC_ARCH_BITS == 64
10716 if (fLongModeGuest)
10717 {
10718 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10719 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10720 }
10721
10722 if ( !fLongModeGuest
10723 && (u32GuestCR4 & X86_CR4_PCIDE))
10724 {
10725 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10726 }
10727
10728 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10729 * 51:32 beyond the processor's physical-address width are 0. */
10730
10731 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10732 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10733 {
10734 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10735 }
10736
10737 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10738 AssertRCBreak(rc);
10739 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10740
10741 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10742 AssertRCBreak(rc);
10743 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10744#endif
10745
10746 /*
10747 * PERF_GLOBAL MSR.
10748 */
10749 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10750 {
10751 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10752 AssertRCBreak(rc);
10753 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10754 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10755 }
10756
10757 /*
10758 * PAT MSR.
10759 */
10760 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10761 {
10762 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10763 AssertRCBreak(rc);
10764 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10765 for (unsigned i = 0; i < 8; i++)
10766 {
10767 uint8_t u8Val = (u64Val & 0xff);
10768 if ( u8Val != 0 /* UC */
10769 && u8Val != 1 /* WC */
10770 && u8Val != 4 /* WT */
10771 && u8Val != 5 /* WP */
10772 && u8Val != 6 /* WB */
10773 && u8Val != 7 /* UC- */)
10774 {
10775 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10776 }
10777 u64Val >>= 8;
10778 }
10779 }
10780
10781 /*
10782 * EFER MSR.
10783 */
10784 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10785 {
10786 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10787 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10788 AssertRCBreak(rc);
10789 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10790 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10791 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10792 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10793 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10794 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10795 || !(u32GuestCR0 & X86_CR0_PG)
10796 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10797 VMX_IGS_EFER_LMA_LME_MISMATCH);
10798 }
10799
10800 /*
10801 * Segment registers.
10802 */
10803 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10804 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10805 if (!(u32Eflags & X86_EFL_VM))
10806 {
10807 /* CS */
10808 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10809 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10810 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10811 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10812 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10813 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10814 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10815 /* CS cannot be loaded with NULL in protected mode. */
10816 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10817 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10818 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10819 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10820 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10821 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10822 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10823 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10824 else
10825 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10826
10827 /* SS */
10828 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10829 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10830 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10831 if ( !(pCtx->cr0 & X86_CR0_PE)
10832 || pCtx->cs.Attr.n.u4Type == 3)
10833 {
10834 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10835 }
10836 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10837 {
10838 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10839 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10840 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10841 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10842 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10843 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10844 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10845 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10846 }
10847
10848 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10849 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10850 {
10851 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10852 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10853 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10854 || pCtx->ds.Attr.n.u4Type > 11
10855 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10856 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10857 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10858 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10859 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10860 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10861 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10862 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10863 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10864 }
10865 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10866 {
10867 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10868 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10869 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10870 || pCtx->es.Attr.n.u4Type > 11
10871 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10872 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10873 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10874 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10875 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10876 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10877 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10878 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10879 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10880 }
10881 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10882 {
10883 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10884 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10885 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10886 || pCtx->fs.Attr.n.u4Type > 11
10887 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10888 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10889 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10890 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10891 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10892 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10893 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10894 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10895 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10896 }
10897 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10898 {
10899 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10900 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10901 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10902 || pCtx->gs.Attr.n.u4Type > 11
10903 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10904 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10905 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10906 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10907 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10908 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10909 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10910 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10911 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10912 }
10913 /* 64-bit capable CPUs. */
10914#if HC_ARCH_BITS == 64
10915 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10916 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10917 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10918 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10919 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10920 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10921 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10922 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10923 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10924 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10925 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10926#endif
10927 }
10928 else
10929 {
10930 /* V86 mode checks. */
10931 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10932 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10933 {
10934 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10935 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10936 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10937 }
10938 else
10939 {
10940 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10941 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10942 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10943 }
10944
10945 /* CS */
10946 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10947 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10948 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10949 /* SS */
10950 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10951 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10952 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10953 /* DS */
10954 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10955 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10956 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10957 /* ES */
10958 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10959 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10960 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10961 /* FS */
10962 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10963 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10964 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10965 /* GS */
10966 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10967 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10968 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10969 /* 64-bit capable CPUs. */
10970#if HC_ARCH_BITS == 64
10971 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10972 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10973 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10974 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10975 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10976 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10977 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10978 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10979 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10980 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10981 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10982#endif
10983 }
10984
10985 /*
10986 * TR.
10987 */
10988 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10989 /* 64-bit capable CPUs. */
10990#if HC_ARCH_BITS == 64
10991 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10992#endif
10993 if (fLongModeGuest)
10994 {
10995 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10996 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10997 }
10998 else
10999 {
11000 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11001 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11002 VMX_IGS_TR_ATTR_TYPE_INVALID);
11003 }
11004 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11005 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11006 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11007 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11008 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11009 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11010 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11011 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11012
11013 /*
11014 * GDTR and IDTR.
11015 */
11016#if HC_ARCH_BITS == 64
11017 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11018 AssertRCBreak(rc);
11019 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11020
11021 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11022 AssertRCBreak(rc);
11023 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11024#endif
11025
11026 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11027 AssertRCBreak(rc);
11028 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11029
11030 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11031 AssertRCBreak(rc);
11032 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11033
11034 /*
11035 * Guest Non-Register State.
11036 */
11037 /* Activity State. */
11038 uint32_t u32ActivityState;
11039 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11040 AssertRCBreak(rc);
11041 HMVMX_CHECK_BREAK( !u32ActivityState
11042 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11043 VMX_IGS_ACTIVITY_STATE_INVALID);
11044 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11045 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11046 uint32_t u32IntrState;
11047 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11048 AssertRCBreak(rc);
11049 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11050 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11051 {
11052 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11053 }
11054
11055 /** @todo Activity state and injecting interrupts. Left as a todo since we
11056 * currently don't use activity states but ACTIVE. */
11057
11058 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11059 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11060
11061 /* Guest interruptibility-state. */
11062 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11063 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11064 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11065 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11066 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11067 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11068 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11069 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11070 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11071 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11072 {
11073 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11074 {
11075 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11076 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11077 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11078 }
11079 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11080 {
11081 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11082 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11083 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11084 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11085 }
11086 }
11087 /** @todo Assumes the processor is not in SMM. */
11088 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11089 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11090 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11091 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11092 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11093 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11094 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11095 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11096 {
11097 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11098 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11099 }
11100
11101 /* Pending debug exceptions. */
11102#if HC_ARCH_BITS == 64
11103 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11104 AssertRCBreak(rc);
11105 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11106 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11107 u32Val = u64Val; /* For pending debug exceptions checks below. */
11108#else
11109 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11110 AssertRCBreak(rc);
11111 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11112 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11113#endif
11114
11115 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11116 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11117 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11118 {
11119 if ( (u32Eflags & X86_EFL_TF)
11120 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11121 {
11122 /* Bit 14 is PendingDebug.BS. */
11123 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11124 }
11125 if ( !(u32Eflags & X86_EFL_TF)
11126 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11127 {
11128 /* Bit 14 is PendingDebug.BS. */
11129 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11130 }
11131 }
11132
11133 /* VMCS link pointer. */
11134 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11135 AssertRCBreak(rc);
11136 if (u64Val != UINT64_C(0xffffffffffffffff))
11137 {
11138 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11139 /** @todo Bits beyond the processor's physical-address width MBZ. */
11140 /** @todo 32-bit located in memory referenced by value of this field (as a
11141 * physical address) must contain the processor's VMCS revision ID. */
11142 /** @todo SMM checks. */
11143 }
11144
11145 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11146 * not using Nested Paging? */
11147 if ( pVM->hm.s.fNestedPaging
11148 && !fLongModeGuest
11149 && CPUMIsGuestInPAEModeEx(pCtx))
11150 {
11151 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11152 AssertRCBreak(rc);
11153 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11154
11155 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11156 AssertRCBreak(rc);
11157 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11158
11159 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11160 AssertRCBreak(rc);
11161 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11162
11163 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11164 AssertRCBreak(rc);
11165 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11166 }
11167
11168 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11169 if (uError == VMX_IGS_ERROR)
11170 uError = VMX_IGS_REASON_NOT_FOUND;
11171 } while (0);
11172
11173 pVCpu->hm.s.u32HMError = uError;
11174 return uError;
11175
11176#undef HMVMX_ERROR_BREAK
11177#undef HMVMX_CHECK_BREAK
11178}
11179
11180/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11181/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11182/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11183
11184/** @name VM-exit handlers.
11185 * @{
11186 */
11187
11188/**
11189 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11190 */
11191HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11192{
11193 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11194 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11195 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11196 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11197 return VINF_SUCCESS;
11198 return VINF_EM_RAW_INTERRUPT;
11199}
11200
11201
11202/**
11203 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11204 */
11205HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11206{
11207 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11208 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11209
11210 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11211 AssertRCReturn(rc, rc);
11212
11213 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11214 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11215 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11216 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11217
11218 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11219 {
11220 /*
11221 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11222 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11223 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11224 *
11225 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11226 */
11227 VMXDispatchHostNmi();
11228 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11229 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11230 return VINF_SUCCESS;
11231 }
11232
11233 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11234 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11235 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11236 { /* likely */ }
11237 else
11238 {
11239 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11240 rcStrictRc1 = VINF_SUCCESS;
11241 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11242 return rcStrictRc1;
11243 }
11244
11245 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11246 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11247 switch (uIntType)
11248 {
11249 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11250 Assert(uVector == X86_XCPT_DB);
11251 /* no break */
11252 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11253 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11254 /* no break */
11255 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11256 {
11257 /*
11258 * If there's any exception caused as a result of event injection, go back to
11259 * the interpreter. The page-fault case is complicated and we manually handle
11260 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11261 * handled in hmR0VmxCheckExitDueToEventDelivery.
11262 */
11263 if (!pVCpu->hm.s.Event.fPending)
11264 { /* likely */ }
11265 else if ( uVector != X86_XCPT_PF
11266 && uVector != X86_XCPT_AC)
11267 {
11268 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11269 rc = VERR_EM_INTERPRETER;
11270 break;
11271 }
11272
11273 switch (uVector)
11274 {
11275 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11276 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11277 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11278 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11279 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11280 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11281 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11282
11283 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11284 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11285 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11286 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11287 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11288 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11289 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11290 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11291 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11292 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11293 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11294 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11295 default:
11296 {
11297 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11298 AssertRCReturn(rc, rc);
11299
11300 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11301 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11302 {
11303 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11304 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11305 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11306
11307 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11308 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11309 AssertRCReturn(rc, rc);
11310 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11311 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11312 0 /* GCPtrFaultAddress */);
11313 AssertRCReturn(rc, rc);
11314 }
11315 else
11316 {
11317 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11318 pVCpu->hm.s.u32HMError = uVector;
11319 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11320 }
11321 break;
11322 }
11323 }
11324 break;
11325 }
11326
11327 default:
11328 {
11329 pVCpu->hm.s.u32HMError = uExitIntInfo;
11330 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11331 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11332 break;
11333 }
11334 }
11335 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11336 return rc;
11337}
11338
11339
11340/**
11341 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11342 */
11343HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11344{
11345 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11346
11347 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11348 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11349
11350 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11352 return VINF_SUCCESS;
11353}
11354
11355
11356/**
11357 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11358 */
11359HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11360{
11361 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11362 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11363 {
11364 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11365 HMVMX_RETURN_UNEXPECTED_EXIT();
11366 }
11367
11368 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11369
11370 /*
11371 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11372 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11373 */
11374 uint32_t uIntrState = 0;
11375 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11376 AssertRCReturn(rc, rc);
11377
11378 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11379 if ( fBlockSti
11380 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11381 {
11382 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11383 }
11384
11385 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11386 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11387
11388 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11389 return VINF_SUCCESS;
11390}
11391
11392
11393/**
11394 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11395 */
11396HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11397{
11398 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11399 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11400 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11401}
11402
11403
11404/**
11405 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11406 */
11407HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11408{
11409 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11410 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11411 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11412}
11413
11414
11415/**
11416 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11417 */
11418HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11419{
11420 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11421 PVM pVM = pVCpu->CTX_SUFF(pVM);
11422 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11423 if (RT_LIKELY(rc == VINF_SUCCESS))
11424 {
11425 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11426 Assert(pVmxTransient->cbInstr == 2);
11427 }
11428 else
11429 {
11430 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11431 rc = VERR_EM_INTERPRETER;
11432 }
11433 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11434 return rc;
11435}
11436
11437
11438/**
11439 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11440 */
11441HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11442{
11443 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11444 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11445 AssertRCReturn(rc, rc);
11446
11447 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11448 return VINF_EM_RAW_EMULATE_INSTR;
11449
11450 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11451 HMVMX_RETURN_UNEXPECTED_EXIT();
11452}
11453
11454
11455/**
11456 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11457 */
11458HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11459{
11460 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11461 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11462 AssertRCReturn(rc, rc);
11463
11464 PVM pVM = pVCpu->CTX_SUFF(pVM);
11465 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11466 if (RT_LIKELY(rc == VINF_SUCCESS))
11467 {
11468 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11469 Assert(pVmxTransient->cbInstr == 2);
11470 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11471 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11472 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11473 }
11474 else
11475 rc = VERR_EM_INTERPRETER;
11476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11477 return rc;
11478}
11479
11480
11481/**
11482 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11483 */
11484HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11485{
11486 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11487 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11488 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11489 AssertRCReturn(rc, rc);
11490
11491 PVM pVM = pVCpu->CTX_SUFF(pVM);
11492 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11493 if (RT_SUCCESS(rc))
11494 {
11495 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11496 Assert(pVmxTransient->cbInstr == 3);
11497 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11498 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11499 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11500 }
11501 else
11502 {
11503 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11504 rc = VERR_EM_INTERPRETER;
11505 }
11506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11507 return rc;
11508}
11509
11510
11511/**
11512 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11513 */
11514HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11515{
11516 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11517 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11518 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11519 AssertRCReturn(rc, rc);
11520
11521 PVM pVM = pVCpu->CTX_SUFF(pVM);
11522 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11523 if (RT_LIKELY(rc == VINF_SUCCESS))
11524 {
11525 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11526 Assert(pVmxTransient->cbInstr == 2);
11527 }
11528 else
11529 {
11530 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11531 rc = VERR_EM_INTERPRETER;
11532 }
11533 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11534 return rc;
11535}
11536
11537
11538/**
11539 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11540 */
11541HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11542{
11543 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11544 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11545
11546 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11547 if (pVCpu->hm.s.fHypercallsEnabled)
11548 {
11549#if 0
11550 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11551#else
11552 /* Aggressive state sync. for now. */
11553 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11554 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11555 AssertRCReturn(rc, rc);
11556#endif
11557
11558 /* Perform the hypercall. */
11559 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11560 if (rcStrict == VINF_SUCCESS)
11561 {
11562 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11563 AssertRCReturn(rc, rc);
11564 }
11565 else
11566 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11567 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11568 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11569
11570 /* If the hypercall changes anything other than guest's general-purpose registers,
11571 we would need to reload the guest changed bits here before VM-entry. */
11572 }
11573 else
11574 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11575
11576 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11577 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11578 {
11579 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11580 rcStrict = VINF_SUCCESS;
11581 }
11582
11583 return rcStrict;
11584}
11585
11586
11587/**
11588 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11589 */
11590HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11591{
11592 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11593 PVM pVM = pVCpu->CTX_SUFF(pVM);
11594 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11595
11596 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11597 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11598 AssertRCReturn(rc, rc);
11599
11600 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11601 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11602 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11603 else
11604 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11605 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11606 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11607 return rcStrict;
11608}
11609
11610
11611/**
11612 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11613 */
11614HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11615{
11616 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11617 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11618 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11619 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11620 AssertRCReturn(rc, rc);
11621
11622 PVM pVM = pVCpu->CTX_SUFF(pVM);
11623 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11624 if (RT_LIKELY(rc == VINF_SUCCESS))
11625 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11626 else
11627 {
11628 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11629 rc = VERR_EM_INTERPRETER;
11630 }
11631 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11632 return rc;
11633}
11634
11635
11636/**
11637 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11638 */
11639HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11640{
11641 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11642 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11643 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11644 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11645 AssertRCReturn(rc, rc);
11646
11647 PVM pVM = pVCpu->CTX_SUFF(pVM);
11648 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11649 rc = VBOXSTRICTRC_VAL(rc2);
11650 if (RT_LIKELY( rc == VINF_SUCCESS
11651 || rc == VINF_EM_HALT))
11652 {
11653 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11654 AssertRCReturn(rc3, rc3);
11655
11656 if ( rc == VINF_EM_HALT
11657 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11658 {
11659 rc = VINF_SUCCESS;
11660 }
11661 }
11662 else
11663 {
11664 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11665 rc = VERR_EM_INTERPRETER;
11666 }
11667 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11668 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11669 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11670 return rc;
11671}
11672
11673
11674/**
11675 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11676 */
11677HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11678{
11679 /*
11680 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11681 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11682 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11683 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11684 */
11685 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11686 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11687 HMVMX_RETURN_UNEXPECTED_EXIT();
11688}
11689
11690
11691/**
11692 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11693 */
11694HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11695{
11696 /*
11697 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11698 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11699 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11700 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11701 */
11702 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11703 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11704 HMVMX_RETURN_UNEXPECTED_EXIT();
11705}
11706
11707
11708/**
11709 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11710 */
11711HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11712{
11713 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11714 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11715 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11716 HMVMX_RETURN_UNEXPECTED_EXIT();
11717}
11718
11719
11720/**
11721 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11722 */
11723HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11724{
11725 /*
11726 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11727 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11728 * See Intel spec. 25.3 "Other Causes of VM-exits".
11729 */
11730 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11731 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11732 HMVMX_RETURN_UNEXPECTED_EXIT();
11733}
11734
11735
11736/**
11737 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11738 * VM-exit.
11739 */
11740HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11741{
11742 /*
11743 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11744 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11745 *
11746 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11747 * See Intel spec. "23.8 Restrictions on VMX operation".
11748 */
11749 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11750 return VINF_SUCCESS;
11751}
11752
11753
11754/**
11755 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11756 * VM-exit.
11757 */
11758HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11759{
11760 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11761 return VINF_EM_RESET;
11762}
11763
11764
11765/**
11766 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11767 */
11768HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11769{
11770 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11771 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11772
11773 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11774 AssertRCReturn(rc, rc);
11775
11776 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11777 rc = VINF_SUCCESS;
11778 else
11779 rc = VINF_EM_HALT;
11780
11781 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11782 if (rc != VINF_SUCCESS)
11783 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11784 return rc;
11785}
11786
11787
11788/**
11789 * VM-exit handler for instructions that result in a \#UD exception delivered to
11790 * the guest.
11791 */
11792HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11793{
11794 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11795 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11796 return VINF_SUCCESS;
11797}
11798
11799
11800/**
11801 * VM-exit handler for expiry of the VMX preemption timer.
11802 */
11803HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11804{
11805 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11806
11807 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11808 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11809
11810 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11811 PVM pVM = pVCpu->CTX_SUFF(pVM);
11812 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11813 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11814 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11815}
11816
11817
11818/**
11819 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11820 */
11821HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11822{
11823 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11824
11825 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11826 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11827 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11828 AssertRCReturn(rc, rc);
11829
11830 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11831 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11832
11833 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11834
11835 return rcStrict;
11836}
11837
11838
11839/**
11840 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11841 */
11842HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11843{
11844 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11845
11846 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11847 /** @todo implement EMInterpretInvpcid() */
11848 return VERR_EM_INTERPRETER;
11849}
11850
11851
11852/**
11853 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11854 * Error VM-exit.
11855 */
11856HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11857{
11858 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11859 AssertRCReturn(rc, rc);
11860
11861 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11862 AssertRCReturn(rc, rc);
11863
11864 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11865 NOREF(uInvalidReason);
11866
11867#ifdef VBOX_STRICT
11868 uint32_t uIntrState;
11869 RTHCUINTREG uHCReg;
11870 uint64_t u64Val;
11871 uint32_t u32Val;
11872
11873 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11874 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11875 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11876 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11877 AssertRCReturn(rc, rc);
11878
11879 Log4(("uInvalidReason %u\n", uInvalidReason));
11880 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11881 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11882 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11883 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11884
11885 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11886 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11887 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11888 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11889 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11890 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11891 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11892 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11893 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11894 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11895 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11896 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11897#else
11898 NOREF(pVmxTransient);
11899#endif
11900
11901 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11902 return VERR_VMX_INVALID_GUEST_STATE;
11903}
11904
11905
11906/**
11907 * VM-exit handler for VM-entry failure due to an MSR-load
11908 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11909 */
11910HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11911{
11912 NOREF(pVmxTransient);
11913 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11914 HMVMX_RETURN_UNEXPECTED_EXIT();
11915}
11916
11917
11918/**
11919 * VM-exit handler for VM-entry failure due to a machine-check event
11920 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11921 */
11922HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11923{
11924 NOREF(pVmxTransient);
11925 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11926 HMVMX_RETURN_UNEXPECTED_EXIT();
11927}
11928
11929
11930/**
11931 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11932 * theory.
11933 */
11934HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11935{
11936 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11937 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11938 return VERR_VMX_UNDEFINED_EXIT_CODE;
11939}
11940
11941
11942/**
11943 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11944 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11945 * Conditional VM-exit.
11946 */
11947HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11948{
11949 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11950
11951 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11953 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11954 return VERR_EM_INTERPRETER;
11955 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11956 HMVMX_RETURN_UNEXPECTED_EXIT();
11957}
11958
11959
11960/**
11961 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11962 */
11963HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11964{
11965 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11966
11967 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11968 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11969 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11970 return VERR_EM_INTERPRETER;
11971 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11972 HMVMX_RETURN_UNEXPECTED_EXIT();
11973}
11974
11975
11976/**
11977 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11978 */
11979HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11980{
11981 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11982
11983 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11984 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11985 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11986 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11987 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11988 {
11989 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11990 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11991 }
11992 AssertRCReturn(rc, rc);
11993 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11994
11995#ifdef VBOX_STRICT
11996 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11997 {
11998 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11999 && pMixedCtx->ecx != MSR_K6_EFER)
12000 {
12001 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12002 pMixedCtx->ecx));
12003 HMVMX_RETURN_UNEXPECTED_EXIT();
12004 }
12005 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12006 {
12007 VMXMSREXITREAD enmRead;
12008 VMXMSREXITWRITE enmWrite;
12009 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12010 AssertRCReturn(rc2, rc2);
12011 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12012 {
12013 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12014 HMVMX_RETURN_UNEXPECTED_EXIT();
12015 }
12016 }
12017 }
12018#endif
12019
12020 PVM pVM = pVCpu->CTX_SUFF(pVM);
12021 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12022 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12023 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12024 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12025 if (RT_SUCCESS(rc))
12026 {
12027 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12028 Assert(pVmxTransient->cbInstr == 2);
12029 }
12030 return rc;
12031}
12032
12033
12034/**
12035 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12036 */
12037HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12038{
12039 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12040 PVM pVM = pVCpu->CTX_SUFF(pVM);
12041 int rc = VINF_SUCCESS;
12042
12043 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12044 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12045 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12046 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12047 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12048 {
12049 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12050 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12051 }
12052 AssertRCReturn(rc, rc);
12053 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12054
12055 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12056 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12057 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12058
12059 if (RT_SUCCESS(rc))
12060 {
12061 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12062
12063 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12064 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12065 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
12066 {
12067 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12068 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12069 EMInterpretWrmsr() changes it. */
12070 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12071 }
12072 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12073 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12074 else if (pMixedCtx->ecx == MSR_K6_EFER)
12075 {
12076 /*
12077 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12078 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12079 * the other bits as well, SCE and NXE. See @bugref{7368}.
12080 */
12081 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12082 }
12083
12084 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12085 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12086 {
12087 switch (pMixedCtx->ecx)
12088 {
12089 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12090 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12091 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12092 case MSR_K8_FS_BASE: /* no break */
12093 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12094 case MSR_K6_EFER: /* already handled above */ break;
12095 default:
12096 {
12097 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12098 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12099 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12100 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12101 break;
12102 }
12103 }
12104 }
12105#ifdef VBOX_STRICT
12106 else
12107 {
12108 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12109 switch (pMixedCtx->ecx)
12110 {
12111 case MSR_IA32_SYSENTER_CS:
12112 case MSR_IA32_SYSENTER_EIP:
12113 case MSR_IA32_SYSENTER_ESP:
12114 case MSR_K8_FS_BASE:
12115 case MSR_K8_GS_BASE:
12116 {
12117 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12118 HMVMX_RETURN_UNEXPECTED_EXIT();
12119 }
12120
12121 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12122 default:
12123 {
12124 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12125 {
12126 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12127 if (pMixedCtx->ecx != MSR_K6_EFER)
12128 {
12129 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12130 pMixedCtx->ecx));
12131 HMVMX_RETURN_UNEXPECTED_EXIT();
12132 }
12133 }
12134
12135 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12136 {
12137 VMXMSREXITREAD enmRead;
12138 VMXMSREXITWRITE enmWrite;
12139 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12140 AssertRCReturn(rc2, rc2);
12141 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12142 {
12143 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12144 HMVMX_RETURN_UNEXPECTED_EXIT();
12145 }
12146 }
12147 break;
12148 }
12149 }
12150 }
12151#endif /* VBOX_STRICT */
12152 }
12153 return rc;
12154}
12155
12156
12157/**
12158 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12159 */
12160HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12161{
12162 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12163
12164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12165 return VINF_EM_RAW_INTERRUPT;
12166}
12167
12168
12169/**
12170 * VM-exit handler for when the TPR value is lowered below the specified
12171 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12172 */
12173HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12174{
12175 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12176 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12177
12178 /*
12179 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12180 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12181 * resume guest execution.
12182 */
12183 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12184 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12185 return VINF_SUCCESS;
12186}
12187
12188
12189/**
12190 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12191 * VM-exit.
12192 *
12193 * @retval VINF_SUCCESS when guest execution can continue.
12194 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12195 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12196 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12197 * interpreter.
12198 */
12199HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12200{
12201 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12202 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12203 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12204 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12205 AssertRCReturn(rc, rc);
12206
12207 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12208 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12209 PVM pVM = pVCpu->CTX_SUFF(pVM);
12210 VBOXSTRICTRC rcStrict;
12211 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12212 switch (uAccessType)
12213 {
12214 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12215 {
12216 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12217 AssertRCReturn(rc, rc);
12218
12219 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12220 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12221 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12222 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12223 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12224 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12225 {
12226 case 0: /* CR0 */
12227 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12228 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12229 break;
12230 case 2: /* CR2 */
12231 /* Nothing to do here, CR2 it's not part of the VMCS. */
12232 break;
12233 case 3: /* CR3 */
12234 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12235 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12236 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12237 break;
12238 case 4: /* CR4 */
12239 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12240 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12241 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12242 break;
12243 case 8: /* CR8 */
12244 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12245 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12246 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12247 break;
12248 default:
12249 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12250 break;
12251 }
12252
12253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12254 break;
12255 }
12256
12257 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12258 {
12259 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12260 AssertRCReturn(rc, rc);
12261
12262 Assert( !pVM->hm.s.fNestedPaging
12263 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12264 || pVCpu->hm.s.fUsingDebugLoop
12265 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12266
12267 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12268 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12269 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12270
12271 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12272 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12273 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12274 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12275 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12276 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12277 VBOXSTRICTRC_VAL(rcStrict)));
12278 break;
12279 }
12280
12281 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12282 {
12283 AssertRCReturn(rc, rc);
12284 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12285 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12286 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12287 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12288 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12289 break;
12290 }
12291
12292 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12293 {
12294 AssertRCReturn(rc, rc);
12295 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12296 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12297 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12298 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12299 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12300 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12301 break;
12302 }
12303
12304 default:
12305 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12306 VERR_VMX_UNEXPECTED_EXCEPTION);
12307 }
12308
12309 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12310 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12311 NOREF(pVM);
12312 return rcStrict;
12313}
12314
12315
12316/**
12317 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12318 * VM-exit.
12319 */
12320HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12321{
12322 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12323 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12324
12325 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12326 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12327 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12328 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12329 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12330 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12331 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12332 AssertRCReturn(rc2, rc2);
12333
12334 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12335 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12336 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12337 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12338 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12339 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12340 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12341 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12342 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12343
12344 /* I/O operation lookup arrays. */
12345 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12346 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12347
12348 VBOXSTRICTRC rcStrict;
12349 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12350 uint32_t const cbInstr = pVmxTransient->cbInstr;
12351 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12352 PVM pVM = pVCpu->CTX_SUFF(pVM);
12353 if (fIOString)
12354 {
12355#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12356 See @bugref{5752#c158}. Should work now. */
12357 /*
12358 * INS/OUTS - I/O String instruction.
12359 *
12360 * Use instruction-information if available, otherwise fall back on
12361 * interpreting the instruction.
12362 */
12363 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12364 fIOWrite ? 'w' : 'r'));
12365 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12366 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12367 {
12368 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12369 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12370 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12371 AssertRCReturn(rc2, rc2);
12372 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12373 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12374 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12375 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12376 if (fIOWrite)
12377 {
12378 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12379 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12380 }
12381 else
12382 {
12383 /*
12384 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12385 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12386 * See Intel Instruction spec. for "INS".
12387 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12388 */
12389 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12390 }
12391 }
12392 else
12393 {
12394 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12395 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12396 AssertRCReturn(rc2, rc2);
12397 rcStrict = IEMExecOne(pVCpu);
12398 }
12399 /** @todo IEM needs to be setting these flags somehow. */
12400 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12401 fUpdateRipAlready = true;
12402#else
12403 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12404 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12405 if (RT_SUCCESS(rcStrict))
12406 {
12407 if (fIOWrite)
12408 {
12409 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12410 (DISCPUMODE)pDis->uAddrMode, cbValue);
12411 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12412 }
12413 else
12414 {
12415 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12416 (DISCPUMODE)pDis->uAddrMode, cbValue);
12417 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12418 }
12419 }
12420 else
12421 {
12422 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12423 pMixedCtx->rip));
12424 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12425 }
12426#endif
12427 }
12428 else
12429 {
12430 /*
12431 * IN/OUT - I/O instruction.
12432 */
12433 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12434 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12435 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12436 if (fIOWrite)
12437 {
12438 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12439 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12440 }
12441 else
12442 {
12443 uint32_t u32Result = 0;
12444 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12445 if (IOM_SUCCESS(rcStrict))
12446 {
12447 /* Save result of I/O IN instr. in AL/AX/EAX. */
12448 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12449 }
12450 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12451 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12452 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12453 }
12454 }
12455
12456 if (IOM_SUCCESS(rcStrict))
12457 {
12458 if (!fUpdateRipAlready)
12459 {
12460 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12461 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12462 }
12463
12464 /*
12465 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12466 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12467 */
12468 if (fIOString)
12469 {
12470 /** @todo Single-step for INS/OUTS with REP prefix? */
12471 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12472 }
12473 else if ( !fDbgStepping
12474 && fGstStepping)
12475 {
12476 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12477 }
12478
12479 /*
12480 * If any I/O breakpoints are armed, we need to check if one triggered
12481 * and take appropriate action.
12482 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12483 */
12484 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12485 AssertRCReturn(rc2, rc2);
12486
12487 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12488 * execution engines about whether hyper BPs and such are pending. */
12489 uint32_t const uDr7 = pMixedCtx->dr[7];
12490 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12491 && X86_DR7_ANY_RW_IO(uDr7)
12492 && (pMixedCtx->cr4 & X86_CR4_DE))
12493 || DBGFBpIsHwIoArmed(pVM)))
12494 {
12495 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12496
12497 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12498 VMMRZCallRing3Disable(pVCpu);
12499 HM_DISABLE_PREEMPT();
12500
12501 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12502
12503 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12504 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12505 {
12506 /* Raise #DB. */
12507 if (fIsGuestDbgActive)
12508 ASMSetDR6(pMixedCtx->dr[6]);
12509 if (pMixedCtx->dr[7] != uDr7)
12510 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12511
12512 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12513 }
12514 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12515 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12516 else if ( rcStrict2 != VINF_SUCCESS
12517 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12518 rcStrict = rcStrict2;
12519 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12520
12521 HM_RESTORE_PREEMPT();
12522 VMMRZCallRing3Enable(pVCpu);
12523 }
12524 }
12525
12526#ifdef VBOX_STRICT
12527 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12528 Assert(!fIOWrite);
12529 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12530 Assert(fIOWrite);
12531 else
12532 {
12533#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12534 * statuses, that the VMM device and some others may return. See
12535 * IOM_SUCCESS() for guidance. */
12536 AssertMsg( RT_FAILURE(rcStrict)
12537 || rcStrict == VINF_SUCCESS
12538 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12539 || rcStrict == VINF_EM_DBG_BREAKPOINT
12540 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12541 || rcStrict == VINF_EM_RAW_TO_R3
12542 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12543#endif
12544 }
12545#endif
12546
12547 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12548 return rcStrict;
12549}
12550
12551
12552/**
12553 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12554 * VM-exit.
12555 */
12556HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12557{
12558 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12559
12560 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12561 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12562 AssertRCReturn(rc, rc);
12563 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12564 {
12565 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12566 AssertRCReturn(rc, rc);
12567 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12568 {
12569 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12570
12571 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12572 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12573
12574 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12575 Assert(!pVCpu->hm.s.Event.fPending);
12576 pVCpu->hm.s.Event.fPending = true;
12577 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12578 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12579 AssertRCReturn(rc, rc);
12580 if (fErrorCodeValid)
12581 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12582 else
12583 pVCpu->hm.s.Event.u32ErrCode = 0;
12584 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12585 && uVector == X86_XCPT_PF)
12586 {
12587 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12588 }
12589
12590 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12591 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12592 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12593 }
12594 }
12595
12596 /* Fall back to the interpreter to emulate the task-switch. */
12597 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12598 return VERR_EM_INTERPRETER;
12599}
12600
12601
12602/**
12603 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12604 */
12605HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12606{
12607 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12608 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12609 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12610 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12611 AssertRCReturn(rc, rc);
12612 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12613 return VINF_EM_DBG_STEPPED;
12614}
12615
12616
12617/**
12618 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12619 */
12620HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12621{
12622 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12623
12624 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12625
12626 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12627 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12628 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12629 {
12630 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12631 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12632 {
12633 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12634 return VERR_EM_INTERPRETER;
12635 }
12636 }
12637 else
12638 {
12639 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12640 rcStrict1 = VINF_SUCCESS;
12641 return rcStrict1;
12642 }
12643
12644#if 0
12645 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12646 * just sync the whole thing. */
12647 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12648#else
12649 /* Aggressive state sync. for now. */
12650 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12651 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12652 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12653#endif
12654 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12655 AssertRCReturn(rc, rc);
12656
12657 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12658 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12659 VBOXSTRICTRC rcStrict2;
12660 switch (uAccessType)
12661 {
12662 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12663 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12664 {
12665 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12666 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12667 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12668
12669 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12670 GCPhys &= PAGE_BASE_GC_MASK;
12671 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12672 PVM pVM = pVCpu->CTX_SUFF(pVM);
12673 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12674 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12675
12676 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12677 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12678 CPUMCTX2CORE(pMixedCtx), GCPhys);
12679 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12680 if ( rcStrict2 == VINF_SUCCESS
12681 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12682 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12683 {
12684 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12685 | HM_CHANGED_GUEST_RSP
12686 | HM_CHANGED_GUEST_RFLAGS
12687 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12688 rcStrict2 = VINF_SUCCESS;
12689 }
12690 break;
12691 }
12692
12693 default:
12694 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12695 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12696 break;
12697 }
12698
12699 if (rcStrict2 != VINF_SUCCESS)
12700 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12701 return rcStrict2;
12702}
12703
12704
12705/**
12706 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12707 * VM-exit.
12708 */
12709HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12710{
12711 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12712
12713 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12714 if (pVmxTransient->fWasGuestDebugStateActive)
12715 {
12716 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12717 HMVMX_RETURN_UNEXPECTED_EXIT();
12718 }
12719
12720 if ( !pVCpu->hm.s.fSingleInstruction
12721 && !pVmxTransient->fWasHyperDebugStateActive)
12722 {
12723 Assert(!DBGFIsStepping(pVCpu));
12724 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12725
12726 /* Don't intercept MOV DRx any more. */
12727 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12728 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12729 AssertRCReturn(rc, rc);
12730
12731 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12732 VMMRZCallRing3Disable(pVCpu);
12733 HM_DISABLE_PREEMPT();
12734
12735 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12736 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12737 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12738
12739 HM_RESTORE_PREEMPT();
12740 VMMRZCallRing3Enable(pVCpu);
12741
12742#ifdef VBOX_WITH_STATISTICS
12743 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12744 AssertRCReturn(rc, rc);
12745 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12746 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12747 else
12748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12749#endif
12750 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12751 return VINF_SUCCESS;
12752 }
12753
12754 /*
12755 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12756 * Update the segment registers and DR7 from the CPU.
12757 */
12758 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12759 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12760 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12761 AssertRCReturn(rc, rc);
12762 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12763
12764 PVM pVM = pVCpu->CTX_SUFF(pVM);
12765 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12766 {
12767 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12768 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12769 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12770 if (RT_SUCCESS(rc))
12771 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12772 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12773 }
12774 else
12775 {
12776 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12777 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12778 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12780 }
12781
12782 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12783 if (RT_SUCCESS(rc))
12784 {
12785 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12786 AssertRCReturn(rc2, rc2);
12787 return VINF_SUCCESS;
12788 }
12789 return rc;
12790}
12791
12792
12793/**
12794 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12795 * Conditional VM-exit.
12796 */
12797HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12798{
12799 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12800 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12801
12802 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12803 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12804 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12805 {
12806 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12807 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12808 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12809 {
12810 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12811 return VERR_EM_INTERPRETER;
12812 }
12813 }
12814 else
12815 {
12816 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12817 rcStrict1 = VINF_SUCCESS;
12818 return rcStrict1;
12819 }
12820
12821 RTGCPHYS GCPhys = 0;
12822 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12823
12824#if 0
12825 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12826#else
12827 /* Aggressive state sync. for now. */
12828 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12829 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12830 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12831#endif
12832 AssertRCReturn(rc, rc);
12833
12834 /*
12835 * If we succeed, resume guest execution.
12836 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12837 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12838 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12839 * weird case. See @bugref{6043}.
12840 */
12841 PVM pVM = pVCpu->CTX_SUFF(pVM);
12842 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12843 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12844 if ( rcStrict2 == VINF_SUCCESS
12845 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12846 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12847 {
12848 /* Successfully handled MMIO operation. */
12849 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12850 | HM_CHANGED_GUEST_RSP
12851 | HM_CHANGED_GUEST_RFLAGS
12852 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12853 return VINF_SUCCESS;
12854 }
12855 return rcStrict2;
12856}
12857
12858
12859/**
12860 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12861 * VM-exit.
12862 */
12863HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12864{
12865 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12866 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12867
12868 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12869 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12870 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12871 {
12872 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12873 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12874 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12875 }
12876 else
12877 {
12878 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12879 rcStrict1 = VINF_SUCCESS;
12880 return rcStrict1;
12881 }
12882
12883 RTGCPHYS GCPhys = 0;
12884 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12885 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12886#if 0
12887 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12888#else
12889 /* Aggressive state sync. for now. */
12890 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12891 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12892 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12893#endif
12894 AssertRCReturn(rc, rc);
12895
12896 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12897 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12898
12899 RTGCUINT uErrorCode = 0;
12900 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12901 uErrorCode |= X86_TRAP_PF_ID;
12902 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12903 uErrorCode |= X86_TRAP_PF_RW;
12904 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12905 uErrorCode |= X86_TRAP_PF_P;
12906
12907 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12908
12909 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12910 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12911
12912 /* Handle the pagefault trap for the nested shadow table. */
12913 PVM pVM = pVCpu->CTX_SUFF(pVM);
12914 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12915 TRPMResetTrap(pVCpu);
12916
12917 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12918 if ( rcStrict2 == VINF_SUCCESS
12919 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12920 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12921 {
12922 /* Successfully synced our nested page tables. */
12923 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12924 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12925 | HM_CHANGED_GUEST_RSP
12926 | HM_CHANGED_GUEST_RFLAGS);
12927 return VINF_SUCCESS;
12928 }
12929
12930 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12931 return rcStrict2;
12932}
12933
12934/** @} */
12935
12936/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12937/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12938/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12939
12940/** @name VM-exit exception handlers.
12941 * @{
12942 */
12943
12944/**
12945 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12946 */
12947static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12948{
12949 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12950 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12951
12952 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12953 AssertRCReturn(rc, rc);
12954
12955 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12956 {
12957 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12958 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12959
12960 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12961 * provides VM-exit instruction length. If this causes problem later,
12962 * disassemble the instruction like it's done on AMD-V. */
12963 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12964 AssertRCReturn(rc2, rc2);
12965 return rc;
12966 }
12967
12968 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12969 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12970 return rc;
12971}
12972
12973
12974/**
12975 * VM-exit exception handler for \#BP (Breakpoint exception).
12976 */
12977static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12978{
12979 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12980 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12981
12982 /** @todo Try optimize this by not saving the entire guest state unless
12983 * really needed. */
12984 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12985 AssertRCReturn(rc, rc);
12986
12987 PVM pVM = pVCpu->CTX_SUFF(pVM);
12988 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12989 if (rc == VINF_EM_RAW_GUEST_TRAP)
12990 {
12991 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12992 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12993 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12994 AssertRCReturn(rc, rc);
12995
12996 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12997 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12998 }
12999
13000 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13001 return rc;
13002}
13003
13004
13005/**
13006 * VM-exit exception handler for \#AC (alignment check exception).
13007 */
13008static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13009{
13010 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13011
13012 /*
13013 * Re-inject it. We'll detect any nesting before getting here.
13014 */
13015 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13016 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13017 AssertRCReturn(rc, rc);
13018 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13019
13020 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13021 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13022 return VINF_SUCCESS;
13023}
13024
13025
13026/**
13027 * VM-exit exception handler for \#DB (Debug exception).
13028 */
13029static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13030{
13031 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13032 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13033 Log6(("XcptDB\n"));
13034
13035 /*
13036 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13037 * for processing.
13038 */
13039 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13040 AssertRCReturn(rc, rc);
13041
13042 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13043 uint64_t uDR6 = X86_DR6_INIT_VAL;
13044 uDR6 |= ( pVmxTransient->uExitQualification
13045 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13046
13047 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13048 if (rc == VINF_EM_RAW_GUEST_TRAP)
13049 {
13050 /*
13051 * The exception was for the guest. Update DR6, DR7.GD and
13052 * IA32_DEBUGCTL.LBR before forwarding it.
13053 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13054 */
13055 VMMRZCallRing3Disable(pVCpu);
13056 HM_DISABLE_PREEMPT();
13057
13058 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13059 pMixedCtx->dr[6] |= uDR6;
13060 if (CPUMIsGuestDebugStateActive(pVCpu))
13061 ASMSetDR6(pMixedCtx->dr[6]);
13062
13063 HM_RESTORE_PREEMPT();
13064 VMMRZCallRing3Enable(pVCpu);
13065
13066 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13067 AssertRCReturn(rc, rc);
13068
13069 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13070 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13071
13072 /* Paranoia. */
13073 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13074 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13075
13076 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13077 AssertRCReturn(rc, rc);
13078
13079 /*
13080 * Raise #DB in the guest.
13081 *
13082 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13083 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13084 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13085 *
13086 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13087 */
13088 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13089 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13090 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13091 AssertRCReturn(rc, rc);
13092 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13093 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13094 return VINF_SUCCESS;
13095 }
13096
13097 /*
13098 * Not a guest trap, must be a hypervisor related debug event then.
13099 * Update DR6 in case someone is interested in it.
13100 */
13101 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13102 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13103 CPUMSetHyperDR6(pVCpu, uDR6);
13104
13105 return rc;
13106}
13107
13108
13109/**
13110 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13111 * point exception).
13112 */
13113static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13114{
13115 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13116
13117 /* We require CR0 and EFER. EFER is always up-to-date. */
13118 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13119 AssertRCReturn(rc, rc);
13120
13121 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13122 VMMRZCallRing3Disable(pVCpu);
13123 HM_DISABLE_PREEMPT();
13124
13125 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
13126 if (pVmxTransient->fWasGuestFPUStateActive)
13127 {
13128 rc = VINF_EM_RAW_GUEST_TRAP;
13129 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13130 }
13131 else
13132 {
13133#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13134 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13135#endif
13136 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13137 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13138 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13139 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13140 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13141 }
13142
13143 HM_RESTORE_PREEMPT();
13144 VMMRZCallRing3Enable(pVCpu);
13145
13146 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13147 {
13148 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13149 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13150 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13151 pVCpu->hm.s.fPreloadGuestFpu = true;
13152 }
13153 else
13154 {
13155 /* Forward #NM to the guest. */
13156 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13157 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13158 AssertRCReturn(rc, rc);
13159 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13160 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13161 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13162 }
13163
13164 return VINF_SUCCESS;
13165}
13166
13167
13168/**
13169 * VM-exit exception handler for \#GP (General-protection exception).
13170 *
13171 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13172 */
13173static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13174{
13175 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13176 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13177
13178 int rc;
13179 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13180 { /* likely */ }
13181 else
13182 {
13183#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13184 Assert(pVCpu->hm.s.fUsingDebugLoop);
13185#endif
13186 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13187 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13188 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13189 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13190 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13191 AssertRCReturn(rc, rc);
13192 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13193 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13194 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13195 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13196 return rc;
13197 }
13198
13199 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13200 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13201
13202 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13203 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13204 AssertRCReturn(rc, rc);
13205
13206 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13207 uint32_t cbOp = 0;
13208 PVM pVM = pVCpu->CTX_SUFF(pVM);
13209 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13210 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13211 if (RT_SUCCESS(rc))
13212 {
13213 rc = VINF_SUCCESS;
13214 Assert(cbOp == pDis->cbInstr);
13215 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13216 switch (pDis->pCurInstr->uOpcode)
13217 {
13218 case OP_CLI:
13219 {
13220 pMixedCtx->eflags.Bits.u1IF = 0;
13221 pMixedCtx->eflags.Bits.u1RF = 0;
13222 pMixedCtx->rip += pDis->cbInstr;
13223 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13224 if ( !fDbgStepping
13225 && pMixedCtx->eflags.Bits.u1TF)
13226 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13228 break;
13229 }
13230
13231 case OP_STI:
13232 {
13233 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13234 pMixedCtx->eflags.Bits.u1IF = 1;
13235 pMixedCtx->eflags.Bits.u1RF = 0;
13236 pMixedCtx->rip += pDis->cbInstr;
13237 if (!fOldIF)
13238 {
13239 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13240 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13241 }
13242 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13243 if ( !fDbgStepping
13244 && pMixedCtx->eflags.Bits.u1TF)
13245 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13246 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13247 break;
13248 }
13249
13250 case OP_HLT:
13251 {
13252 rc = VINF_EM_HALT;
13253 pMixedCtx->rip += pDis->cbInstr;
13254 pMixedCtx->eflags.Bits.u1RF = 0;
13255 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13257 break;
13258 }
13259
13260 case OP_POPF:
13261 {
13262 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13263 uint32_t cbParm;
13264 uint32_t uMask;
13265 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13266 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13267 {
13268 cbParm = 4;
13269 uMask = 0xffffffff;
13270 }
13271 else
13272 {
13273 cbParm = 2;
13274 uMask = 0xffff;
13275 }
13276
13277 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13278 RTGCPTR GCPtrStack = 0;
13279 X86EFLAGS Eflags;
13280 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13281 &GCPtrStack);
13282 if (RT_SUCCESS(rc))
13283 {
13284 Assert(sizeof(Eflags.u32) >= cbParm);
13285 Eflags.u32 = 0;
13286 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13287 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13288 }
13289 if (RT_FAILURE(rc))
13290 {
13291 rc = VERR_EM_INTERPRETER;
13292 break;
13293 }
13294 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13295 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13296 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13297 pMixedCtx->esp += cbParm;
13298 pMixedCtx->esp &= uMask;
13299 pMixedCtx->rip += pDis->cbInstr;
13300 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13301 | HM_CHANGED_GUEST_RSP
13302 | HM_CHANGED_GUEST_RFLAGS);
13303 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13304 POPF restores EFLAGS.TF. */
13305 if ( !fDbgStepping
13306 && fGstStepping)
13307 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13308 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13309 break;
13310 }
13311
13312 case OP_PUSHF:
13313 {
13314 uint32_t cbParm;
13315 uint32_t uMask;
13316 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13317 {
13318 cbParm = 4;
13319 uMask = 0xffffffff;
13320 }
13321 else
13322 {
13323 cbParm = 2;
13324 uMask = 0xffff;
13325 }
13326
13327 /* Get the stack pointer & push the contents of eflags onto the stack. */
13328 RTGCPTR GCPtrStack = 0;
13329 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13330 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13331 if (RT_FAILURE(rc))
13332 {
13333 rc = VERR_EM_INTERPRETER;
13334 break;
13335 }
13336 X86EFLAGS Eflags = pMixedCtx->eflags;
13337 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13338 Eflags.Bits.u1RF = 0;
13339 Eflags.Bits.u1VM = 0;
13340
13341 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13342 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13343 {
13344 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13345 rc = VERR_EM_INTERPRETER;
13346 break;
13347 }
13348 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13349 pMixedCtx->esp -= cbParm;
13350 pMixedCtx->esp &= uMask;
13351 pMixedCtx->rip += pDis->cbInstr;
13352 pMixedCtx->eflags.Bits.u1RF = 0;
13353 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13354 | HM_CHANGED_GUEST_RSP
13355 | HM_CHANGED_GUEST_RFLAGS);
13356 if ( !fDbgStepping
13357 && pMixedCtx->eflags.Bits.u1TF)
13358 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13359 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13360 break;
13361 }
13362
13363 case OP_IRET:
13364 {
13365 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13366 * instruction reference. */
13367 RTGCPTR GCPtrStack = 0;
13368 uint32_t uMask = 0xffff;
13369 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13370 uint16_t aIretFrame[3];
13371 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13372 {
13373 rc = VERR_EM_INTERPRETER;
13374 break;
13375 }
13376 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13377 &GCPtrStack);
13378 if (RT_SUCCESS(rc))
13379 {
13380 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13381 PGMACCESSORIGIN_HM));
13382 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13383 }
13384 if (RT_FAILURE(rc))
13385 {
13386 rc = VERR_EM_INTERPRETER;
13387 break;
13388 }
13389 pMixedCtx->eip = 0;
13390 pMixedCtx->ip = aIretFrame[0];
13391 pMixedCtx->cs.Sel = aIretFrame[1];
13392 pMixedCtx->cs.ValidSel = aIretFrame[1];
13393 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13394 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13395 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13396 pMixedCtx->sp += sizeof(aIretFrame);
13397 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13398 | HM_CHANGED_GUEST_SEGMENT_REGS
13399 | HM_CHANGED_GUEST_RSP
13400 | HM_CHANGED_GUEST_RFLAGS);
13401 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13402 if ( !fDbgStepping
13403 && fGstStepping)
13404 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13405 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13406 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13407 break;
13408 }
13409
13410 case OP_INT:
13411 {
13412 uint16_t uVector = pDis->Param1.uValue & 0xff;
13413 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13414 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13415 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13416 break;
13417 }
13418
13419 case OP_INTO:
13420 {
13421 if (pMixedCtx->eflags.Bits.u1OF)
13422 {
13423 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13424 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13425 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13426 }
13427 else
13428 {
13429 pMixedCtx->eflags.Bits.u1RF = 0;
13430 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13431 }
13432 break;
13433 }
13434
13435 default:
13436 {
13437 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13438 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13439 EMCODETYPE_SUPERVISOR);
13440 rc = VBOXSTRICTRC_VAL(rc2);
13441 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13442 /** @todo We have to set pending-debug exceptions here when the guest is
13443 * single-stepping depending on the instruction that was interpreted. */
13444 Log4(("#GP rc=%Rrc\n", rc));
13445 break;
13446 }
13447 }
13448 }
13449 else
13450 rc = VERR_EM_INTERPRETER;
13451
13452 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13453 ("#GP Unexpected rc=%Rrc\n", rc));
13454 return rc;
13455}
13456
13457
13458/**
13459 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13460 * the exception reported in the VMX transient structure back into the VM.
13461 *
13462 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13463 * up-to-date.
13464 */
13465static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13466{
13467 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13468#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13469 Assert(pVCpu->hm.s.fUsingDebugLoop);
13470#endif
13471
13472 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13473 hmR0VmxCheckExitDueToEventDelivery(). */
13474 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13475 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13476 AssertRCReturn(rc, rc);
13477 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13478
13479#ifdef DEBUG_ramshankar
13480 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13481 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13482 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13483#endif
13484
13485 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13486 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13487 return VINF_SUCCESS;
13488}
13489
13490
13491/**
13492 * VM-exit exception handler for \#PF (Page-fault exception).
13493 */
13494static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13495{
13496 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13497 PVM pVM = pVCpu->CTX_SUFF(pVM);
13498 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13499 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13500 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13501 AssertRCReturn(rc, rc);
13502
13503 if (!pVM->hm.s.fNestedPaging)
13504 { /* likely */ }
13505 else
13506 {
13507#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13508 Assert(pVCpu->hm.s.fUsingDebugLoop);
13509#endif
13510 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13511 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13512 {
13513 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13514 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13515 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13516 }
13517 else
13518 {
13519 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13520 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13521 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13522 }
13523 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13524 return rc;
13525 }
13526
13527 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13528 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13529 if (pVmxTransient->fVectoringPF)
13530 {
13531 Assert(pVCpu->hm.s.Event.fPending);
13532 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13533 }
13534
13535 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13536 AssertRCReturn(rc, rc);
13537
13538 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13539 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13540
13541 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13542 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13543 (RTGCPTR)pVmxTransient->uExitQualification);
13544
13545 Log4(("#PF: rc=%Rrc\n", rc));
13546 if (rc == VINF_SUCCESS)
13547 {
13548#if 0
13549 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13550 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13551 * memory? We don't update the whole state here... */
13552 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13553 | HM_CHANGED_GUEST_RSP
13554 | HM_CHANGED_GUEST_RFLAGS
13555 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13556#else
13557 /*
13558 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13559 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13560 */
13561 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13562 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13563#endif
13564 TRPMResetTrap(pVCpu);
13565 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13566 return rc;
13567 }
13568
13569 if (rc == VINF_EM_RAW_GUEST_TRAP)
13570 {
13571 if (!pVmxTransient->fVectoringDoublePF)
13572 {
13573 /* It's a guest page fault and needs to be reflected to the guest. */
13574 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13575 TRPMResetTrap(pVCpu);
13576 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13577 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13578 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13579 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13580 }
13581 else
13582 {
13583 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13584 TRPMResetTrap(pVCpu);
13585 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13586 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13587 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13588 }
13589
13590 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13591 return VINF_SUCCESS;
13592 }
13593
13594 TRPMResetTrap(pVCpu);
13595 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13596 return rc;
13597}
13598
13599/** @} */
13600
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette