VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 62320

Last change on this file since 62320 was 62320, checked in by vboxsync, 9 years ago

VMX: Update uVmcsState when changing VMX state.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 582.3 KB
Line 
1/* $Id: HMVMXR0.cpp 62320 2016-07-19 11:41:29Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#ifdef VBOX_WITH_NEW_APIC
38# include <VBox/vmm/apic.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_SWAP_FPU_STATE
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name Updated-guest-state flags.
71 * @{ */
72#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93 | HMVMX_UPDATED_GUEST_RSP \
94 | HMVMX_UPDATED_GUEST_RFLAGS \
95 | HMVMX_UPDATED_GUEST_CR0 \
96 | HMVMX_UPDATED_GUEST_CR3 \
97 | HMVMX_UPDATED_GUEST_CR4 \
98 | HMVMX_UPDATED_GUEST_GDTR \
99 | HMVMX_UPDATED_GUEST_IDTR \
100 | HMVMX_UPDATED_GUEST_LDTR \
101 | HMVMX_UPDATED_GUEST_TR \
102 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103 | HMVMX_UPDATED_GUEST_DEBUG \
104 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
109 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110 | HMVMX_UPDATED_GUEST_INTR_STATE \
111 | HMVMX_UPDATED_GUEST_APIC_STATE)
112/** @} */
113
114/** @name
115 * Flags to skip redundant reads of some common VMCS fields that are not part of
116 * the guest-CPU state but are in the transient structure.
117 */
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually except:
143 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145 * due to bugs in Intel CPUs.
146 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
147 * support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*********************************************************************************************************************************
198* Structures and Typedefs *
199*********************************************************************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG fEFlags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit code qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u7Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns Strict VBox status code (i.e. informational status codes too).
323 * @param pVCpu The cross context virtual CPU structure.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
345 * out-of-sync. Make sure to update the required
346 * fields before using them.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368 bool fStepping, uint32_t *puIntState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381
382/** @name VM-exit handlers.
383 * @{
384 */
385static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397static FNVMXEXITHANDLER hmR0VmxExitHlt;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413static FNVMXEXITHANDLER hmR0VmxExitMwait;
414static FNVMXEXITHANDLER hmR0VmxExitMtf;
415static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416static FNVMXEXITHANDLER hmR0VmxExitPause;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443/*********************************************************************************************************************************
444* Global Variables *
445*********************************************************************************************************************************/
446#ifdef HMVMX_USE_FUNCTION_TABLE
447
448/**
449 * VMX_EXIT dispatch table.
450 */
451static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452{
453 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493 /* 40 UNDEFINED */ hmR0VmxExitPause,
494 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
515 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518};
519#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521#ifdef VBOX_STRICT
522static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523{
524 /* 0 */ "(Not Used)",
525 /* 1 */ "VMCALL executed in VMX root operation.",
526 /* 2 */ "VMCLEAR with invalid physical address.",
527 /* 3 */ "VMCLEAR with VMXON pointer.",
528 /* 4 */ "VMLAUNCH with non-clear VMCS.",
529 /* 5 */ "VMRESUME with non-launched VMCS.",
530 /* 6 */ "VMRESUME after VMXOFF",
531 /* 7 */ "VM-entry with invalid control fields.",
532 /* 8 */ "VM-entry with invalid host state fields.",
533 /* 9 */ "VMPTRLD with invalid physical address.",
534 /* 10 */ "VMPTRLD with VMXON pointer.",
535 /* 11 */ "VMPTRLD with incorrect revision identifier.",
536 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537 /* 13 */ "VMWRITE to read-only VMCS component.",
538 /* 14 */ "(Not Used)",
539 /* 15 */ "VMXON executed in VMX root operation.",
540 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541 /* 17 */ "VM-entry with non-launched executing VMCS.",
542 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543 /* 19 */ "VMCALL with non-clear VMCS.",
544 /* 20 */ "VMCALL with invalid VM-exit control fields.",
545 /* 21 */ "(Not Used)",
546 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548 /* 24 */ "VMCALL with invalid SMM-monitor features.",
549 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550 /* 26 */ "VM-entry with events blocked by MOV SS.",
551 /* 27 */ "(Not Used)",
552 /* 28 */ "Invalid operand to INVEPT/INVVPID."
553};
554#endif /* VBOX_STRICT */
555
556
557
558/**
559 * Updates the VM's last error record.
560 *
561 * If there was a VMX instruction error, reads the error data from the VMCS and
562 * updates VCPU's last error record as well.
563 *
564 * @param pVM The cross context VM structure.
565 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
566 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567 * VERR_VMX_INVALID_VMCS_FIELD.
568 * @param rc The error code.
569 */
570static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571{
572 AssertPtr(pVM);
573 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574 || rc == VERR_VMX_UNABLE_TO_START_VM)
575 {
576 AssertPtrReturnVoid(pVCpu);
577 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578 }
579 pVM->hm.s.lLastError = rc;
580}
581
582
583/**
584 * Reads the VM-entry interruption-information field from the VMCS into the VMX
585 * transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit interruption-information field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 */
641DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642{
643 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644 {
645 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646 AssertRCReturn(rc, rc);
647 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648 }
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-exit interruption error code from the VMCS into the VMX
655 * transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 */
660DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661{
662 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663 {
664 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665 AssertRCReturn(rc, rc);
666 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667 }
668 return VINF_SUCCESS;
669}
670
671
672/**
673 * Reads the VM-exit instruction length field from the VMCS into the VMX
674 * transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVmxTransient Pointer to the VMX transient structure.
678 */
679DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680{
681 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682 {
683 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684 AssertRCReturn(rc, rc);
685 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686 }
687 return VINF_SUCCESS;
688}
689
690
691/**
692 * Reads the VM-exit instruction-information field from the VMCS into
693 * the VMX transient structure.
694 *
695 * @returns VBox status code.
696 * @param pVmxTransient Pointer to the VMX transient structure.
697 */
698DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699{
700 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701 {
702 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703 AssertRCReturn(rc, rc);
704 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705 }
706 return VINF_SUCCESS;
707}
708
709
710/**
711 * Reads the exit code qualification from the VMCS into the VMX transient
712 * structure.
713 *
714 * @returns VBox status code.
715 * @param pVCpu The cross context virtual CPU structure of the
716 * calling EMT. (Required for the VMCS cache case.)
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722 {
723 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the IDT-vectoring information field from the VMCS into the VMX
733 * transient structure.
734 *
735 * @returns VBox status code.
736 * @param pVmxTransient Pointer to the VMX transient structure.
737 *
738 * @remarks No-long-jump zone!!!
739 */
740DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743 {
744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring error code from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 */
759DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760{
761 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762 {
763 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764 AssertRCReturn(rc, rc);
765 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766 }
767 return VINF_SUCCESS;
768}
769
770
771/**
772 * Enters VMX root mode operation on the current CPU.
773 *
774 * @returns VBox status code.
775 * @param pVM The cross context VM structure. Can be
776 * NULL, after a resume.
777 * @param HCPhysCpuPage Physical address of the VMXON region.
778 * @param pvCpuPage Pointer to the VMXON region.
779 */
780static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781{
782 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784 Assert(pvCpuPage);
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 if (pVM)
788 {
789 /* Write the VMCS revision dword to the VMXON region. */
790 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791 }
792
793 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794 RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796 /* Enable the VMX bit in CR4 if necessary. */
797 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
798
799 /* Enter VMX root mode. */
800 int rc = VMXEnable(HCPhysCpuPage);
801 if (RT_FAILURE(rc))
802 {
803 if (!(uOldCr4 & X86_CR4_VMXE))
804 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806 if (pVM)
807 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808 }
809
810 /* Restore interrupts. */
811 ASMSetFlags(fEFlags);
812 return rc;
813}
814
815
816/**
817 * Exits VMX root mode operation on the current CPU.
818 *
819 * @returns VBox status code.
820 */
821static int hmR0VmxLeaveRootMode(void)
822{
823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826 RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828 /* If we're for some reason not in VMX root mode, then don't leave it. */
829 RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831 int rc;
832 if (uHostCR4 & X86_CR4_VMXE)
833 {
834 /* Exit VMX root mode and clear the VMX bit in CR4. */
835 VMXDisable();
836 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837 rc = VINF_SUCCESS;
838 }
839 else
840 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842 /* Restore interrupts. */
843 ASMSetFlags(fEFlags);
844 return rc;
845}
846
847
848/**
849 * Allocates and maps one physically contiguous page. The allocated page is
850 * zero'd out. (Used by various VT-x structures).
851 *
852 * @returns IPRT status code.
853 * @param pMemObj Pointer to the ring-0 memory object.
854 * @param ppVirt Where to store the virtual address of the
855 * allocation.
856 * @param pHCPhys Where to store the physical address of the
857 * allocation.
858 */
859DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860{
861 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866 if (RT_FAILURE(rc))
867 return rc;
868 *ppVirt = RTR0MemObjAddress(*pMemObj);
869 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
870 ASMMemZero32(*ppVirt, PAGE_SIZE);
871 return VINF_SUCCESS;
872}
873
874
875/**
876 * Frees and unmaps an allocated physical page.
877 *
878 * @param pMemObj Pointer to the ring-0 memory object.
879 * @param ppVirt Where to re-initialize the virtual address of
880 * allocation as 0.
881 * @param pHCPhys Where to re-initialize the physical address of the
882 * allocation as 0.
883 */
884DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885{
886 AssertPtr(pMemObj);
887 AssertPtr(ppVirt);
888 AssertPtr(pHCPhys);
889 if (*pMemObj != NIL_RTR0MEMOBJ)
890 {
891 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
892 AssertRC(rc);
893 *pMemObj = NIL_RTR0MEMOBJ;
894 *ppVirt = 0;
895 *pHCPhys = 0;
896 }
897}
898
899
900/**
901 * Worker function to free VT-x related structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM The cross context VM structure.
905 */
906static void hmR0VmxStructsFree(PVM pVM)
907{
908 for (VMCPUID i = 0; i < pVM->cCpus; i++)
909 {
910 PVMCPU pVCpu = &pVM->aCpus[i];
911 AssertPtr(pVCpu);
912
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921 }
922
923 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926#endif
927}
928
929
930/**
931 * Worker function to allocate VT-x related VM structures.
932 *
933 * @returns IPRT status code.
934 * @param pVM The cross context VM structure.
935 */
936static int hmR0VmxStructsAlloc(PVM pVM)
937{
938 /*
939 * Initialize members up-front so we can cleanup properly on allocation failure.
940 */
941#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944 pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953#endif
954 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#endif
1516 return false;
1517}
1518
1519
1520/**
1521 * Saves a set of guest MSRs back into the guest-CPU context.
1522 *
1523 * @param pVCpu The cross context virtual CPU structure.
1524 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525 * out-of-sync. Make sure to update the required fields
1526 * before using them.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1536 {
1537 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538#if HC_ARCH_BITS == 64
1539 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1540 {
1541 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1542 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1543 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1544 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1545 }
1546#endif
1547 }
1548}
1549
1550
1551/**
1552 * Loads a set of guests MSRs to allow read/passthru to the guest.
1553 *
1554 * The name of this function is slightly confusing. This function does NOT
1555 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1556 * common prefix for functions dealing with "lazy restoration" of the shared
1557 * MSRs.
1558 *
1559 * @param pVCpu The cross context virtual CPU structure.
1560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1561 * out-of-sync. Make sure to update the required fields
1562 * before using them.
1563 *
1564 * @remarks No-long-jump zone!!!
1565 */
1566static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1567{
1568 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1569 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1570
1571#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1572 do { \
1573 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1574 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1575 else \
1576 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1577 } while (0)
1578
1579 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1581 {
1582#if HC_ARCH_BITS == 64
1583 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1584 {
1585 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1586 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1587 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1588 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1589 }
1590#endif
1591 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1592 }
1593
1594#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1595}
1596
1597
1598/**
1599 * Performs lazy restoration of the set of host MSRs if they were previously
1600 * loaded with guest MSR values.
1601 *
1602 * @param pVCpu The cross context virtual CPU structure.
1603 *
1604 * @remarks No-long-jump zone!!!
1605 * @remarks The guest MSRs should have been saved back into the guest-CPU
1606 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1607 */
1608static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1609{
1610 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1612
1613 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1614 {
1615 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1616#if HC_ARCH_BITS == 64
1617 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1618 {
1619 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1620 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1621 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1622 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1623 }
1624#endif
1625 }
1626 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1627}
1628
1629
1630/**
1631 * Verifies that our cached values of the VMCS controls are all
1632 * consistent with what's actually present in the VMCS.
1633 *
1634 * @returns VBox status code.
1635 * @param pVCpu The cross context virtual CPU structure.
1636 */
1637static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1638{
1639 uint32_t u32Val;
1640 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1643 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1648 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1654
1655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1656 AssertRCReturn(rc, rc);
1657 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1658 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1659
1660 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661 {
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1666 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1667 }
1668
1669 return VINF_SUCCESS;
1670}
1671
1672
1673#ifdef VBOX_STRICT
1674/**
1675 * Verifies that our cached host EFER value has not changed
1676 * since we cached it.
1677 *
1678 * @param pVCpu The cross context virtual CPU structure.
1679 */
1680static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1681{
1682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1685 {
1686 uint64_t u64Val;
1687 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1688 AssertRC(rc);
1689
1690 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1691 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1692 }
1693}
1694
1695
1696/**
1697 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1698 * VMCS are correct.
1699 *
1700 * @param pVCpu The cross context virtual CPU structure.
1701 */
1702static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1703{
1704 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1705
1706 /* Verify MSR counts in the VMCS are what we think it should be. */
1707 uint32_t cMsrs;
1708 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1718 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1719 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1720 {
1721 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1722 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1723 pGuestMsr->u32Msr, cMsrs));
1724
1725 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1726 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1727 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1728
1729 /* Verify that the permissions are as expected in the MSR bitmap. */
1730 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1731 {
1732 VMXMSREXITREAD enmRead;
1733 VMXMSREXITWRITE enmWrite;
1734 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1735 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1736 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1739 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1740 }
1741 else
1742 {
1743 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1744 pGuestMsr->u32Msr, cMsrs));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1746 pGuestMsr->u32Msr, cMsrs));
1747 }
1748 }
1749 }
1750}
1751#endif /* VBOX_STRICT */
1752
1753
1754/**
1755 * Flushes the TLB using EPT.
1756 *
1757 * @returns VBox status code.
1758 * @param pVCpu The cross context virtual CPU structure of the calling
1759 * EMT. Can be NULL depending on @a enmFlush.
1760 * @param enmFlush Type of flush.
1761 *
1762 * @remarks Caller is responsible for making sure this function is called only
1763 * when NestedPaging is supported and providing @a enmFlush that is
1764 * supported by the CPU.
1765 * @remarks Can be called with interrupts disabled.
1766 */
1767static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1768{
1769 uint64_t au64Descriptor[2];
1770 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1771 au64Descriptor[0] = 0;
1772 else
1773 {
1774 Assert(pVCpu);
1775 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1776 }
1777 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1778
1779 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1780 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1781 rc));
1782 if ( RT_SUCCESS(rc)
1783 && pVCpu)
1784 {
1785 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1786 }
1787}
1788
1789
1790/**
1791 * Flushes the TLB using VPID.
1792 *
1793 * @returns VBox status code.
1794 * @param pVM The cross context VM structure.
1795 * @param pVCpu The cross context virtual CPU structure of the calling
1796 * EMT. Can be NULL depending on @a enmFlush.
1797 * @param enmFlush Type of flush.
1798 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1799 * on @a enmFlush).
1800 *
1801 * @remarks Can be called with interrupts disabled.
1802 */
1803static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804{
1805 NOREF(pVM);
1806 AssertPtr(pVM);
1807 Assert(pVM->hm.s.vmx.fVpid);
1808
1809 uint64_t au64Descriptor[2];
1810 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1811 {
1812 au64Descriptor[0] = 0;
1813 au64Descriptor[1] = 0;
1814 }
1815 else
1816 {
1817 AssertPtr(pVCpu);
1818 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1819 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1820 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1821 au64Descriptor[1] = GCPtr;
1822 }
1823
1824 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1825 AssertMsg(rc == VINF_SUCCESS,
1826 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1827 if ( RT_SUCCESS(rc)
1828 && pVCpu)
1829 {
1830 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1831 }
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by guest virtual address. Only relevant for
1837 * EPT/VPID, otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM The cross context VM structure.
1841 * @param pVCpu The cross context virtual CPU structure.
1842 * @param GCVirt Guest virtual address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1845{
1846 AssertPtr(pVM);
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1855 * See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1858 * function maybe called in a loop with individual addresses.
1859 */
1860 if (pVM->hm.s.vmx.fVpid)
1861 {
1862 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1863 {
1864 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1880 * otherwise there is nothing really to invalidate.
1881 *
1882 * @returns VBox status code.
1883 * @param pVM The cross context VM structure.
1884 * @param pVCpu The cross context virtual CPU structure.
1885 * @param GCPhys Guest physical address of the page to invalidate.
1886 */
1887VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1888{
1889 NOREF(pVM); NOREF(GCPhys);
1890 LogFlowFunc(("%RGp\n", GCPhys));
1891
1892 /*
1893 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1894 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1895 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1896 */
1897 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1899 return VINF_SUCCESS;
1900}
1901
1902
1903/**
1904 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1905 * case where neither EPT nor VPID is supported by the CPU.
1906 *
1907 * @param pVM The cross context VM structure.
1908 * @param pVCpu The cross context virtual CPU structure.
1909 * @param pCpu Pointer to the global HM struct.
1910 *
1911 * @remarks Called with interrupts disabled.
1912 */
1913static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914{
1915 AssertPtr(pVCpu);
1916 AssertPtr(pCpu);
1917 NOREF(pVM);
1918
1919 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924 pVCpu->hm.s.fForceTLBFlush = false;
1925 return;
1926}
1927
1928
1929/**
1930 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931 *
1932 * @param pVM The cross context VM structure.
1933 * @param pVCpu The cross context virtual CPU structure.
1934 * @param pCpu Pointer to the global HM CPU struct.
1935 * @remarks All references to "ASID" in this function pertains to "VPID" in
1936 * Intel's nomenclature. The reason is, to avoid confusion in compare
1937 * statements since the host-CPU copies are named "ASID".
1938 *
1939 * @remarks Called with interrupts disabled.
1940 */
1941static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942{
1943#ifdef VBOX_WITH_STATISTICS
1944 bool fTlbFlushed = false;
1945# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947 if (!fTlbFlushed) \
1948 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949 } while (0)
1950#else
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953#endif
1954
1955 AssertPtr(pVM);
1956 AssertPtr(pCpu);
1957 AssertPtr(pVCpu);
1958 Assert(pCpu->idCpu != NIL_RTCPUID);
1959
1960 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964 /*
1965 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1966 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1967 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1968 */
1969 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971 {
1972 ++pCpu->uCurrentAsid;
1973 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974 {
1975 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978 }
1979
1980 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984 /*
1985 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987 */
1988 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1989 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990 HMVMX_SET_TAGGED_TLB_FLUSHED();
1991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1992 }
1993
1994 /* Check for explicit TLB flushes. */
1995 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1996 {
1997 /*
1998 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1999 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2000 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2001 * but not guest-physical mappings.
2002 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2003 */
2004 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006 HMVMX_SET_TAGGED_TLB_FLUSHED();
2007 }
2008
2009 pVCpu->hm.s.fForceTLBFlush = false;
2010 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022 /* Update VMCS with the VPID. */
2023 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024 AssertRC(rc);
2025
2026#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027}
2028
2029
2030/**
2031 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032 *
2033 * @returns VBox status code.
2034 * @param pVM The cross context VM structure.
2035 * @param pVCpu The cross context virtual CPU structure.
2036 * @param pCpu Pointer to the global HM CPU struct.
2037 *
2038 * @remarks Called with interrupts disabled.
2039 */
2040static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2041{
2042 AssertPtr(pVM);
2043 AssertPtr(pVCpu);
2044 AssertPtr(pCpu);
2045 Assert(pCpu->idCpu != NIL_RTCPUID);
2046 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2047 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2048
2049 /*
2050 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2051 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2052 */
2053 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2054 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2055 {
2056 pVCpu->hm.s.fForceTLBFlush = true;
2057 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2058 }
2059
2060 /* Check for explicit TLB flushes. */
2061 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2062 {
2063 pVCpu->hm.s.fForceTLBFlush = true;
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2065 }
2066
2067 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2068 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2069
2070 if (pVCpu->hm.s.fForceTLBFlush)
2071 {
2072 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2073 pVCpu->hm.s.fForceTLBFlush = false;
2074 }
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM The cross context VM structure.
2083 * @param pVCpu The cross context virtual CPU structure.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151
2152 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2153 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2154 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2155 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2156 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2158 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2159
2160 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2161 AssertRC(rc);
2162}
2163
2164
2165/**
2166 * Flushes the guest TLB entry based on CPU capabilities.
2167 *
2168 * @param pVCpu The cross context virtual CPU structure.
2169 * @param pCpu Pointer to the global HM CPU struct.
2170 */
2171DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2172{
2173#ifdef HMVMX_ALWAYS_FLUSH_TLB
2174 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2175#endif
2176 PVM pVM = pVCpu->CTX_SUFF(pVM);
2177 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2178 {
2179 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2180 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2181 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2182 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2183 default:
2184 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2185 break;
2186 }
2187
2188 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2189}
2190
2191
2192/**
2193 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2194 * TLB entries from the host TLB before VM-entry.
2195 *
2196 * @returns VBox status code.
2197 * @param pVM The cross context VM structure.
2198 */
2199static int hmR0VmxSetupTaggedTlb(PVM pVM)
2200{
2201 /*
2202 * Determine optimal flush type for Nested Paging.
2203 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2204 * guest execution (see hmR3InitFinalizeR0()).
2205 */
2206 if (pVM->hm.s.fNestedPaging)
2207 {
2208 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2209 {
2210 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2212 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221
2222 /* Make sure the write-back cacheable memory type for EPT is supported. */
2223 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2224 {
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* EPT requires a page-walk length of 4. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237 }
2238 else
2239 {
2240 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246
2247 /*
2248 * Determine optimal flush type for VPID.
2249 */
2250 if (pVM->hm.s.vmx.fVpid)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2262 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2264 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2266 pVM->hm.s.vmx.fVpid = false;
2267 }
2268 }
2269 else
2270 {
2271 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2272 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277
2278 /*
2279 * Setup the handler for flushing tagged-TLBs.
2280 */
2281 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2282 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2283 else if (pVM->hm.s.fNestedPaging)
2284 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2285 else if (pVM->hm.s.vmx.fVpid)
2286 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2287 else
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2289 return VINF_SUCCESS;
2290}
2291
2292
2293/**
2294 * Sets up pin-based VM-execution controls in the VMCS.
2295 *
2296 * @returns VBox status code.
2297 * @param pVM The cross context VM structure.
2298 * @param pVCpu The cross context virtual CPU structure.
2299 */
2300static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2301{
2302 AssertPtr(pVM);
2303 AssertPtr(pVCpu);
2304
2305 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2306 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2307
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2309 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2310
2311 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2312 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2313
2314 /* Enable the VMX preemption timer. */
2315 if (pVM->hm.s.vmx.fUsePreemptTimer)
2316 {
2317 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2319 }
2320
2321#ifdef VBOX_WITH_NEW_APIC
2322#if 0
2323 /* Enable posted-interrupt processing. */
2324 if (pVM->hm.s.fPostedIntrs)
2325 {
2326 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329 }
2330#endif
2331#endif
2332
2333 if ((val & zap) != val)
2334 {
2335 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2336 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2337 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2338 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2339 }
2340
2341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2342 AssertRCReturn(rc, rc);
2343
2344 pVCpu->hm.s.vmx.u32PinCtls = val;
2345 return rc;
2346}
2347
2348
2349/**
2350 * Sets up processor-based VM-execution controls in the VMCS.
2351 *
2352 * @returns VBox status code.
2353 * @param pVM The cross context VM structure.
2354 * @param pVCpu The cross context virtual CPU structure.
2355 */
2356static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2357{
2358 AssertPtr(pVM);
2359 AssertPtr(pVCpu);
2360
2361 int rc = VERR_INTERNAL_ERROR_5;
2362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2364
2365 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2368 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2369 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2372
2373 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2374 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2375 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2376 {
2377 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2378 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2379 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2380 }
2381
2382 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2383 if (!pVM->hm.s.fNestedPaging)
2384 {
2385 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2386 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2387 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2389 }
2390
2391 /* Use TPR shadowing if supported by the CPU. */
2392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2393 {
2394 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2397 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2398 AssertRCReturn(rc, rc);
2399
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2401 /* CR8 writes cause a VM-exit based on TPR threshold. */
2402 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2403 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2404 }
2405 else
2406 {
2407 /*
2408 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2409 * Set this control only for 64-bit guests.
2410 */
2411 if (pVM->hm.s.fAllow64BitGuests)
2412 {
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2414 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2415 }
2416 }
2417
2418 /* Use MSR-bitmaps if supported by the CPU. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2420 {
2421 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2422
2423 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2424 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2425 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2426 AssertRCReturn(rc, rc);
2427
2428 /*
2429 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2430 * automatically using dedicated fields in the VMCS.
2431 */
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2434 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2435 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437
2438#if HC_ARCH_BITS == 64
2439 /*
2440 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2441 */
2442 if (pVM->hm.s.fAllow64BitGuests)
2443 {
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 }
2449#endif
2450 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2451 }
2452
2453 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2454 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2455 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2456
2457 if ((val & zap) != val)
2458 {
2459 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2460 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2461 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2462 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463 }
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2466 AssertRCReturn(rc, rc);
2467
2468 pVCpu->hm.s.vmx.u32ProcCtls = val;
2469
2470 /*
2471 * Secondary processor-based VM-execution controls.
2472 */
2473 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2474 {
2475 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2476 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2477
2478 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2479 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2480
2481 if (pVM->hm.s.fNestedPaging)
2482 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2483 else
2484 {
2485 /*
2486 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2487 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2488 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2489 */
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2492 }
2493
2494 if (pVM->hm.s.vmx.fVpid)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2496
2497 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2498 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2499
2500#ifdef VBOX_WITH_NEW_APIC
2501#if 0
2502 if (pVM->hm.s.fVirtApicRegs)
2503 {
2504 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2506
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2509 }
2510#endif
2511#endif
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2529 && pVM->hm.s.vmx.cPleGapTicks
2530 && pVM->hm.s.vmx.cPleWindowTicks)
2531 {
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2535 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2536 AssertRCReturn(rc, rc);
2537 }
2538
2539 if ((val & zap) != val)
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2542 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2548 AssertRCReturn(rc, rc);
2549
2550 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2551 }
2552 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553 {
2554 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2555 "available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 return VINF_SUCCESS;
2561}
2562
2563
2564/**
2565 * Sets up miscellaneous (everything other than Pin & Processor-based
2566 * VM-execution) control fields in the VMCS.
2567 *
2568 * @returns VBox status code.
2569 * @param pVM The cross context VM structure.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2573{
2574 NOREF(pVM);
2575 AssertPtr(pVM);
2576 AssertPtr(pVCpu);
2577
2578 int rc = VERR_GENERAL_FAILURE;
2579
2580 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581#if 0
2582 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586 /*
2587 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589 * We thus use the exception bitmap to control it rather than use both.
2590 */
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594 /** @todo Explore possibility of using IO-bitmaps. */
2595 /* All IO & IOIO instructions cause VM-exits. */
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2598
2599 /* Initialize the MSR-bitmap area. */
2600 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2601 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2603 AssertRCReturn(rc, rc);
2604#endif
2605
2606 /* Setup MSR auto-load/store area. */
2607 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 AssertRCReturn(rc, rc);
2617
2618 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2620 AssertRCReturn(rc, rc);
2621
2622 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2623#if 0
2624 /* Setup debug controls */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2626 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2627 AssertRCReturn(rc, rc);
2628#endif
2629
2630 return rc;
2631}
2632
2633
2634/**
2635 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2636 *
2637 * @returns VBox status code.
2638 * @param pVM The cross context VM structure.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2642{
2643 AssertPtr(pVM);
2644 AssertPtr(pVCpu);
2645
2646 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2647
2648 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2649
2650 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2652
2653 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2654 and writes, and because recursive #DBs can cause the CPU hang, we must always
2655 intercept #DB. */
2656 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2657
2658 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2659 if (!pVM->hm.s.fNestedPaging)
2660 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2661
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2664 AssertRCReturn(rc, rc);
2665 return rc;
2666}
2667
2668
2669/**
2670 * Sets up the initial guest-state mask. The guest-state mask is consulted
2671 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2672 * for the nested virtualization case (as it would cause a VM-exit).
2673 *
2674 * @param pVCpu The cross context virtual CPU structure.
2675 */
2676static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2677{
2678 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2679 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2740 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791 /* Set revision dword at the beginning of the VMCS structure. */
2792 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 /* Load this VMCS as the current VMCS. */
2800 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824#if HC_ARCH_BITS == 32
2825 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828#endif
2829
2830 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838 }
2839
2840 return VINF_SUCCESS;
2841}
2842
2843
2844/**
2845 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846 * the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVM The cross context VM structure.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853{
2854 NOREF(pVM); NOREF(pVCpu);
2855
2856 RTCCUINTREG uReg = ASMGetCR0();
2857 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858 AssertRCReturn(rc, rc);
2859
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 AssertRCReturn(rc, rc);
2863
2864 uReg = ASMGetCR4();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866 AssertRCReturn(rc, rc);
2867 return rc;
2868}
2869
2870
2871#if HC_ARCH_BITS == 64
2872/**
2873 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874 * requirements. See hmR0VmxSaveHostSegmentRegs().
2875 */
2876# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2878 { \
2879 bool fValidSelector = true; \
2880 if ((selValue) & X86_SEL_LDT) \
2881 { \
2882 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884 } \
2885 if (fValidSelector) \
2886 { \
2887 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2888 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889 } \
2890 (selValue) = 0; \
2891 }
2892#endif
2893
2894
2895/**
2896 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897 * the host-state area in the VMCS.
2898 *
2899 * @returns VBox status code.
2900 * @param pVM The cross context VM structure.
2901 * @param pVCpu The cross context virtual CPU structure.
2902 */
2903DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904{
2905 int rc = VERR_INTERNAL_ERROR_5;
2906
2907#if HC_ARCH_BITS == 64
2908 /*
2909 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2910 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2911 *
2912 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2913 * Was observed booting Solaris10u10 32-bit guest.
2914 */
2915 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2916 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2917 {
2918 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2919 pVCpu->idCpu));
2920 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2921 }
2922 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2923#endif
2924
2925 /*
2926 * Host DS, ES, FS and GS segment registers.
2927 */
2928#if HC_ARCH_BITS == 64
2929 RTSEL uSelDS = ASMGetDS();
2930 RTSEL uSelES = ASMGetES();
2931 RTSEL uSelFS = ASMGetFS();
2932 RTSEL uSelGS = ASMGetGS();
2933#else
2934 RTSEL uSelDS = 0;
2935 RTSEL uSelES = 0;
2936 RTSEL uSelFS = 0;
2937 RTSEL uSelGS = 0;
2938#endif
2939
2940 /*
2941 * Host CS and SS segment registers.
2942 */
2943 RTSEL uSelCS = ASMGetCS();
2944 RTSEL uSelSS = ASMGetSS();
2945
2946 /*
2947 * Host TR segment register.
2948 */
2949 RTSEL uSelTR = ASMGetTR();
2950
2951#if HC_ARCH_BITS == 64
2952 /*
2953 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2954 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2955 */
2956 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2957 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2958 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2959 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2960# undef VMXLOCAL_ADJUST_HOST_SEG
2961#endif
2962
2963 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2964 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2965 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2966 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2967 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2968 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2969 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2970 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2971 Assert(uSelCS);
2972 Assert(uSelTR);
2973
2974 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2975#if 0
2976 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2977 Assert(uSelSS != 0);
2978#endif
2979
2980 /* Write these host selector fields into the host-state area in the VMCS. */
2981 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2983#if HC_ARCH_BITS == 64
2984 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2985 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2986 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2987 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2988#else
2989 NOREF(uSelDS);
2990 NOREF(uSelES);
2991 NOREF(uSelFS);
2992 NOREF(uSelGS);
2993#endif
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2995 AssertRCReturn(rc, rc);
2996
2997 /*
2998 * Host GDTR and IDTR.
2999 */
3000 RTGDTR Gdtr;
3001 RTIDTR Idtr;
3002 RT_ZERO(Gdtr);
3003 RT_ZERO(Idtr);
3004 ASMGetGDTR(&Gdtr);
3005 ASMGetIDTR(&Idtr);
3006 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3007 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3008 AssertRCReturn(rc, rc);
3009
3010#if HC_ARCH_BITS == 64
3011 /*
3012 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3013 * maximum limit (0xffff) on every VM-exit.
3014 */
3015 if (Gdtr.cbGdt != 0xffff)
3016 {
3017 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3018 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3019 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3020 }
3021
3022 /*
3023 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3024 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3025 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3026 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3027 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3028 * hosts where we are pretty sure it won't cause trouble.
3029 */
3030# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3031 if (Idtr.cbIdt < 0x0fff)
3032# else
3033 if (Idtr.cbIdt != 0xffff)
3034# endif
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3037 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3039 }
3040#endif
3041
3042 /*
3043 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3044 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3045 */
3046 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3047 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3048 VERR_VMX_INVALID_HOST_STATE);
3049
3050 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3051#if HC_ARCH_BITS == 64
3052 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3053
3054 /*
3055 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3056 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3057 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3058 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3059 *
3060 * [1] See Intel spec. 3.5 "System Descriptor Types".
3061 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3062 */
3063 Assert(pDesc->System.u4Type == 11);
3064 if ( pDesc->System.u16LimitLow != 0x67
3065 || pDesc->System.u4LimitHigh)
3066 {
3067 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3068 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3069 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3071 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3072
3073 /* Store the GDTR here as we need it while restoring TR. */
3074 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3075 }
3076#else
3077 NOREF(pVM);
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return rc;
3100}
3101
3102
3103/**
3104 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3105 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3106 * the host after every successful VM-exit.
3107 *
3108 * @returns VBox status code.
3109 * @param pVM The cross context VM structure.
3110 * @param pVCpu The cross context virtual CPU structure.
3111 *
3112 * @remarks No-long-jump zone!!!
3113 */
3114DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3115{
3116 NOREF(pVM);
3117
3118 AssertPtr(pVCpu);
3119 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3120
3121 /*
3122 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3123 * rather than swapping them on every VM-entry.
3124 */
3125 hmR0VmxLazySaveHostMsrs(pVCpu);
3126
3127 /*
3128 * Host Sysenter MSRs.
3129 */
3130 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3131#if HC_ARCH_BITS == 32
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3133 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3134#else
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3136 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3137#endif
3138 AssertRCReturn(rc, rc);
3139
3140 /*
3141 * Host EFER MSR.
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3143 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3144 */
3145 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3146 {
3147 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3148 AssertRCReturn(rc, rc);
3149 }
3150
3151 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3152 * hmR0VmxLoadGuestExitCtls() !! */
3153
3154 return rc;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3163 * hmR0VMxLoadGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3168 * out-of-sync. Make sure to update the required fields
3169 * before using them.
3170 *
3171 * @remarks Requires EFER, CR4.
3172 * @remarks No-long-jump zone!!!
3173 */
3174static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3175{
3176#ifdef HMVMX_ALWAYS_SWAP_EFER
3177 return true;
3178#endif
3179
3180#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3181 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3182 if (CPUMIsGuestInLongMode(pVCpu))
3183 return false;
3184#endif
3185
3186 PVM pVM = pVCpu->CTX_SUFF(pVM);
3187 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3188 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3189
3190 /*
3191 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3192 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3193 */
3194 if ( CPUMIsGuestInLongMode(pVCpu)
3195 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3196 {
3197 return true;
3198 }
3199
3200 /*
3201 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3202 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3203 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3204 */
3205 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3206 && (pMixedCtx->cr0 & X86_CR0_PG)
3207 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3208 {
3209 /* Assert that host is PAE capable. */
3210 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3211 return true;
3212 }
3213
3214 /** @todo Check the latest Intel spec. for any other bits,
3215 * like SMEP/SMAP? */
3216 return false;
3217}
3218
3219
3220/**
3221 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3222 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3223 * controls".
3224 *
3225 * @returns VBox status code.
3226 * @param pVCpu The cross context virtual CPU structure.
3227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3228 * out-of-sync. Make sure to update the required fields
3229 * before using them.
3230 *
3231 * @remarks Requires EFER.
3232 * @remarks No-long-jump zone!!!
3233 */
3234DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3235{
3236 int rc = VINF_SUCCESS;
3237 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3238 {
3239 PVM pVM = pVCpu->CTX_SUFF(pVM);
3240 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3241 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3242
3243 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3244 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3245
3246 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3247 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3248 {
3249 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3250 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3251 }
3252 else
3253 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3254
3255 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3256 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3257 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3258 {
3259 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3260 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3261 }
3262
3263 /*
3264 * The following should -not- be set (since we're not in SMM mode):
3265 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3266 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3267 */
3268
3269 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3270 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3271
3272 if ((val & zap) != val)
3273 {
3274 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3275 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3276 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3277 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3278 }
3279
3280 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3281 AssertRCReturn(rc, rc);
3282
3283 pVCpu->hm.s.vmx.u32EntryCtls = val;
3284 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3285 }
3286 return rc;
3287}
3288
3289
3290/**
3291 * Sets up the VM-exit controls in the VMCS.
3292 *
3293 * @returns VBox status code.
3294 * @param pVCpu The cross context virtual CPU structure.
3295 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3296 * out-of-sync. Make sure to update the required fields
3297 * before using them.
3298 *
3299 * @remarks Requires EFER.
3300 */
3301DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3302{
3303 NOREF(pMixedCtx);
3304
3305 int rc = VINF_SUCCESS;
3306 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3307 {
3308 PVM pVM = pVCpu->CTX_SUFF(pVM);
3309 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3310 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3311
3312 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3313 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3314
3315 /*
3316 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3317 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3318 */
3319#if HC_ARCH_BITS == 64
3320 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3321 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3322#else
3323 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3324 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3325 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3326 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3327 {
3328 /* The switcher returns to long mode, EFER is managed by the switcher. */
3329 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3330 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3331 }
3332 else
3333 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3334#endif
3335
3336 /* If the newer VMCS fields for managing EFER exists, use it. */
3337 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3338 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3339 {
3340 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3341 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3342 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3343 }
3344
3345 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3346 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3347
3348 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3349 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3350 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3351
3352 if ( pVM->hm.s.vmx.fUsePreemptTimer
3353 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3354 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3355
3356 if ((val & zap) != val)
3357 {
3358 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3359 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3362 }
3363
3364 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3365 AssertRCReturn(rc, rc);
3366
3367 pVCpu->hm.s.vmx.u32ExitCtls = val;
3368 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3369 }
3370 return rc;
3371}
3372
3373
3374/**
3375 * Sets the TPR threshold in the VMCS.
3376 *
3377 * @returns VBox status code.
3378 * @param pVCpu The cross context virtual CPU structure.
3379 * @param u32TprThreshold The TPR threshold (task-priority class only).
3380 */
3381DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3382{
3383 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3384 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
3385 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3386}
3387
3388
3389/**
3390 * Loads the guest APIC and related state.
3391 *
3392 * @returns VBox status code.
3393 * @param pVCpu The cross context virtual CPU structure.
3394 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3395 * out-of-sync. Make sure to update the required fields
3396 * before using them.
3397 */
3398DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3399{
3400 NOREF(pMixedCtx);
3401
3402 int rc = VINF_SUCCESS;
3403 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3404 {
3405 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3418 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3419 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3420 * the interrupt when we VM-exit for other reasons.
3421 */
3422 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3423 uint32_t u32TprThreshold = 0;
3424 if (fPendingIntr)
3425 {
3426 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3427 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3428 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3429 if (u8PendingPriority <= u8TprPriority)
3430 u32TprThreshold = u8PendingPriority;
3431 else
3432 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3433 }
3434
3435 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3436 AssertRCReturn(rc, rc);
3437 }
3438
3439 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3440 }
3441 return rc;
3442}
3443
3444
3445/**
3446 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3447 *
3448 * @returns Guest's interruptibility-state.
3449 * @param pVCpu The cross context virtual CPU structure.
3450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3451 * out-of-sync. Make sure to update the required fields
3452 * before using them.
3453 *
3454 * @remarks No-long-jump zone!!!
3455 */
3456DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3457{
3458 /*
3459 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3460 */
3461 uint32_t uIntrState = 0;
3462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3463 {
3464 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3465 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3466 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3467 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pMixedCtx->eflags.Bits.u1IF)
3470 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3478 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3479 */
3480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3481 }
3482 }
3483
3484 /*
3485 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3486 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3487 * setting this would block host-NMIs and IRET will not clear the blocking.
3488 *
3489 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3490 */
3491 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3492 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3493 {
3494 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3495 }
3496
3497 return uIntrState;
3498}
3499
3500
3501/**
3502 * Loads the guest's interruptibility-state into the guest-state area in the
3503 * VMCS.
3504 *
3505 * @returns VBox status code.
3506 * @param pVCpu The cross context virtual CPU structure.
3507 * @param uIntrState The interruptibility-state to set.
3508 */
3509static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3510{
3511 NOREF(pVCpu);
3512 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3513 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3514 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3515 AssertRC(rc);
3516 return rc;
3517}
3518
3519
3520/**
3521 * Loads the exception intercepts required for guest execution in the VMCS.
3522 *
3523 * @returns VBox status code.
3524 * @param pVCpu The cross context virtual CPU structure.
3525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3526 * out-of-sync. Make sure to update the required fields
3527 * before using them.
3528 */
3529static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3530{
3531 NOREF(pMixedCtx);
3532 int rc = VINF_SUCCESS;
3533 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3534 {
3535 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3536 if (pVCpu->hm.s.fGIMTrapXcptUD)
3537 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3538#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3539 else
3540 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3541#endif
3542
3543 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3544 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3545
3546 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3547 AssertRCReturn(rc, rc);
3548
3549 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3550 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3551 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3552 }
3553 return rc;
3554}
3555
3556
3557/**
3558 * Loads the guest's RIP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu The cross context virtual CPU structure.
3562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3563 * out-of-sync. Make sure to update the required fields
3564 * before using them.
3565 *
3566 * @remarks No-long-jump zone!!!
3567 */
3568static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3569{
3570 int rc = VINF_SUCCESS;
3571 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3572 {
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3574 AssertRCReturn(rc, rc);
3575
3576 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3577 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3578 HMCPU_CF_VALUE(pVCpu)));
3579 }
3580 return rc;
3581}
3582
3583
3584/**
3585 * Loads the guest's RSP into the guest-state area in the VMCS.
3586 *
3587 * @returns VBox status code.
3588 * @param pVCpu The cross context virtual CPU structure.
3589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590 * out-of-sync. Make sure to update the required fields
3591 * before using them.
3592 *
3593 * @remarks No-long-jump zone!!!
3594 */
3595static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596{
3597 int rc = VINF_SUCCESS;
3598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3599 {
3600 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3601 AssertRCReturn(rc, rc);
3602
3603 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3604 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3605 }
3606 return rc;
3607}
3608
3609
3610/**
3611 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu The cross context virtual CPU structure.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 int rc = VINF_SUCCESS;
3624 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3625 {
3626 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3627 Let us assert it as such and use 32-bit VMWRITE. */
3628 Assert(!(pMixedCtx->rflags.u64 >> 32));
3629 X86EFLAGS Eflags = pMixedCtx->eflags;
3630 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3631 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3632 * These will never be cleared/set, unless some other part of the VMM
3633 * code is buggy - in which case we're better of finding and fixing
3634 * those bugs than hiding them. */
3635 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3636 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3637 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3638 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3639
3640 /*
3641 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3642 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3643 */
3644 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3645 {
3646 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3647 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3648 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3649 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3650 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3651 }
3652
3653 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3654 AssertRCReturn(rc, rc);
3655
3656 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3657 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3658 }
3659 return rc;
3660}
3661
3662
3663/**
3664 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3665 *
3666 * @returns VBox status code.
3667 * @param pVCpu The cross context virtual CPU structure.
3668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3669 * out-of-sync. Make sure to update the required fields
3670 * before using them.
3671 *
3672 * @remarks No-long-jump zone!!!
3673 */
3674DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3675{
3676 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3677 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3678 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3679 AssertRCReturn(rc, rc);
3680 return rc;
3681}
3682
3683
3684/**
3685 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3686 * CR0 is partially shared with the host and we have to consider the FPU bits.
3687 *
3688 * @returns VBox status code.
3689 * @param pVCpu The cross context virtual CPU structure.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 {
3777 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3778 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3779 }
3780 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3781
3782 if (fInterceptNM)
3783 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3784 else
3785 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3786
3787 if (fInterceptMF)
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3789 else
3790 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3791
3792 /* Additional intercepts for debugging, define these yourself explicitly. */
3793#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3794 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3795 | RT_BIT(X86_XCPT_BP)
3796 | RT_BIT(X86_XCPT_DE)
3797 | RT_BIT(X86_XCPT_NM)
3798 | RT_BIT(X86_XCPT_TS)
3799 | RT_BIT(X86_XCPT_UD)
3800 | RT_BIT(X86_XCPT_NP)
3801 | RT_BIT(X86_XCPT_SS)
3802 | RT_BIT(X86_XCPT_GP)
3803 | RT_BIT(X86_XCPT_PF)
3804 | RT_BIT(X86_XCPT_MF)
3805 ;
3806#elif defined(HMVMX_ALWAYS_TRAP_PF)
3807 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3808#endif
3809
3810 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3811
3812 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3813 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3814 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3815 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3816 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3817 else
3818 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3819
3820 u32GuestCR0 |= uSetCR0;
3821 u32GuestCR0 &= uZapCR0;
3822 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3823
3824 /* Write VT-x's view of the guest CR0 into the VMCS. */
3825 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3826 AssertRCReturn(rc, rc);
3827 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3828 uZapCR0));
3829
3830 /*
3831 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3832 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3833 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3834 */
3835 uint32_t u32CR0Mask = 0;
3836 u32CR0Mask = X86_CR0_PE
3837 | X86_CR0_NE
3838 | X86_CR0_WP
3839 | X86_CR0_PG
3840 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3841 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3842 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3843
3844 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3845 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3846 * and @bugref{6944}. */
3847#if 0
3848 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3849 u32CR0Mask &= ~X86_CR0_PE;
3850#endif
3851 if (pVM->hm.s.fNestedPaging)
3852 u32CR0Mask &= ~X86_CR0_WP;
3853
3854 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3855 if (fInterceptNM)
3856 {
3857 u32CR0Mask |= X86_CR0_TS
3858 | X86_CR0_MP;
3859 }
3860
3861 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3862 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3863 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3864 AssertRCReturn(rc, rc);
3865 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3866
3867 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3868 }
3869 return rc;
3870}
3871
3872
3873/**
3874 * Loads the guest control registers (CR3, CR4) into the guest-state area
3875 * in the VMCS.
3876 *
3877 * @returns VBox strict status code.
3878 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3879 * without unrestricted guest access and the VMMDev is not presently
3880 * mapped (e.g. EFI32).
3881 *
3882 * @param pVCpu The cross context virtual CPU structure.
3883 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3884 * out-of-sync. Make sure to update the required fields
3885 * before using them.
3886 *
3887 * @remarks No-long-jump zone!!!
3888 */
3889static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3890{
3891 int rc = VINF_SUCCESS;
3892 PVM pVM = pVCpu->CTX_SUFF(pVM);
3893
3894 /*
3895 * Guest CR2.
3896 * It's always loaded in the assembler code. Nothing to do here.
3897 */
3898
3899 /*
3900 * Guest CR3.
3901 */
3902 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3903 {
3904 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3905 if (pVM->hm.s.fNestedPaging)
3906 {
3907 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3908
3909 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3910 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3911 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3912 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3913
3914 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3915 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3916 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3917
3918 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3919 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3920 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3921 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3922 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3923 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3924 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3925
3926 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3927 AssertRCReturn(rc, rc);
3928 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3929
3930 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3931 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3932 {
3933 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3934 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3935 {
3936 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3937 AssertRCReturn(rc, rc);
3938 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3939 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3940 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3941 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3942 AssertRCReturn(rc, rc);
3943 }
3944
3945 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3946 have Unrestricted Execution to handle the guest when it's not using paging. */
3947 GCPhysGuestCR3 = pMixedCtx->cr3;
3948 }
3949 else
3950 {
3951 /*
3952 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3953 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3954 * EPT takes care of translating it to host-physical addresses.
3955 */
3956 RTGCPHYS GCPhys;
3957 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3958
3959 /* We obtain it here every time as the guest could have relocated this PCI region. */
3960 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3961 if (RT_SUCCESS(rc))
3962 { /* likely */ }
3963 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3964 {
3965 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3966 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3967 }
3968 else
3969 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3970
3971 GCPhysGuestCR3 = GCPhys;
3972 }
3973
3974 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3975 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3976 }
3977 else
3978 {
3979 /* Non-nested paging case, just use the hypervisor's CR3. */
3980 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3981
3982 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3983 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3984 }
3985 AssertRCReturn(rc, rc);
3986
3987 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3988 }
3989
3990 /*
3991 * Guest CR4.
3992 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3993 */
3994 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3995 {
3996 Assert(!(pMixedCtx->cr4 >> 32));
3997 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3998
3999 /* The guest's view of its CR4 is unblemished. */
4000 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4001 AssertRCReturn(rc, rc);
4002 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4003
4004 /* Setup VT-x's view of the guest CR4. */
4005 /*
4006 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4007 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4008 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4009 */
4010 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4011 {
4012 Assert(pVM->hm.s.vmx.pRealModeTSS);
4013 Assert(PDMVmmDevHeapIsEnabled(pVM));
4014 u32GuestCR4 &= ~X86_CR4_VME;
4015 }
4016
4017 if (pVM->hm.s.fNestedPaging)
4018 {
4019 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4020 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4021 {
4022 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4023 u32GuestCR4 |= X86_CR4_PSE;
4024 /* Our identity mapping is a 32-bit page directory. */
4025 u32GuestCR4 &= ~X86_CR4_PAE;
4026 }
4027 /* else use guest CR4.*/
4028 }
4029 else
4030 {
4031 /*
4032 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4033 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4034 */
4035 switch (pVCpu->hm.s.enmShadowMode)
4036 {
4037 case PGMMODE_REAL: /* Real-mode. */
4038 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4039 case PGMMODE_32_BIT: /* 32-bit paging. */
4040 {
4041 u32GuestCR4 &= ~X86_CR4_PAE;
4042 break;
4043 }
4044
4045 case PGMMODE_PAE: /* PAE paging. */
4046 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4047 {
4048 u32GuestCR4 |= X86_CR4_PAE;
4049 break;
4050 }
4051
4052 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4053 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4054#ifdef VBOX_ENABLE_64_BITS_GUESTS
4055 break;
4056#endif
4057 default:
4058 AssertFailed();
4059 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4060 }
4061 }
4062
4063 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4064 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4065 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4066 u32GuestCR4 |= uSetCR4;
4067 u32GuestCR4 &= uZapCR4;
4068
4069 /* Write VT-x's view of the guest CR4 into the VMCS. */
4070 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4071 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4072 AssertRCReturn(rc, rc);
4073
4074 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4075 uint32_t u32CR4Mask = X86_CR4_VME
4076 | X86_CR4_PAE
4077 | X86_CR4_PGE
4078 | X86_CR4_PSE
4079 | X86_CR4_VMXE;
4080 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4081 u32CR4Mask |= X86_CR4_OSXSAVE;
4082 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4083 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4084 AssertRCReturn(rc, rc);
4085
4086 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4087 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4088
4089 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4090 }
4091 return rc;
4092}
4093
4094
4095/**
4096 * Loads the guest debug registers into the guest-state area in the VMCS.
4097 *
4098 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4099 *
4100 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4101 *
4102 * @returns VBox status code.
4103 * @param pVCpu The cross context virtual CPU structure.
4104 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4105 * out-of-sync. Make sure to update the required fields
4106 * before using them.
4107 *
4108 * @remarks No-long-jump zone!!!
4109 */
4110static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4111{
4112 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4113 return VINF_SUCCESS;
4114
4115#ifdef VBOX_STRICT
4116 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4117 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4118 {
4119 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4120 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4121 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4122 }
4123#endif
4124
4125 int rc;
4126 PVM pVM = pVCpu->CTX_SUFF(pVM);
4127 bool fSteppingDB = false;
4128 bool fInterceptMovDRx = false;
4129 if (pVCpu->hm.s.fSingleInstruction)
4130 {
4131 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4132 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4133 {
4134 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4136 AssertRCReturn(rc, rc);
4137 Assert(fSteppingDB == false);
4138 }
4139 else
4140 {
4141 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4142 pVCpu->hm.s.fClearTrapFlag = true;
4143 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4144 fSteppingDB = true;
4145 }
4146 }
4147
4148 if ( fSteppingDB
4149 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4150 {
4151 /*
4152 * Use the combined guest and host DRx values found in the hypervisor
4153 * register set because the debugger has breakpoints active or someone
4154 * is single stepping on the host side without a monitor trap flag.
4155 *
4156 * Note! DBGF expects a clean DR6 state before executing guest code.
4157 */
4158#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4159 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4160 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4161 {
4162 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4163 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4164 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4165 }
4166 else
4167#endif
4168 if (!CPUMIsHyperDebugStateActive(pVCpu))
4169 {
4170 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4171 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4172 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4173 }
4174
4175 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4176 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4177 AssertRCReturn(rc, rc);
4178
4179 pVCpu->hm.s.fUsingHyperDR7 = true;
4180 fInterceptMovDRx = true;
4181 }
4182 else
4183 {
4184 /*
4185 * If the guest has enabled debug registers, we need to load them prior to
4186 * executing guest code so they'll trigger at the right time.
4187 */
4188 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4189 {
4190#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4191 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4192 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4193 {
4194 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4195 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4196 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4197 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4198 }
4199 else
4200#endif
4201 if (!CPUMIsGuestDebugStateActive(pVCpu))
4202 {
4203 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4204 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4205 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4206 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4207 }
4208 Assert(!fInterceptMovDRx);
4209 }
4210 /*
4211 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4212 * must intercept #DB in order to maintain a correct DR6 guest value, and
4213 * because we need to intercept it to prevent nested #DBs from hanging the
4214 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4215 */
4216#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4217 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4218 && !CPUMIsGuestDebugStateActive(pVCpu))
4219#else
4220 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4221#endif
4222 {
4223 fInterceptMovDRx = true;
4224 }
4225
4226 /* Update guest DR7. */
4227 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4228 AssertRCReturn(rc, rc);
4229
4230 pVCpu->hm.s.fUsingHyperDR7 = false;
4231 }
4232
4233 /*
4234 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4235 */
4236 if (fInterceptMovDRx)
4237 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4238 else
4239 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4241 AssertRCReturn(rc, rc);
4242
4243 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4244 return VINF_SUCCESS;
4245}
4246
4247
4248#ifdef VBOX_STRICT
4249/**
4250 * Strict function to validate segment registers.
4251 *
4252 * @remarks ASSUMES CR0 is up to date.
4253 */
4254static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4255{
4256 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4257 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4258 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4259 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4260 && ( !CPUMIsGuestInRealModeEx(pCtx)
4261 && !CPUMIsGuestInV86ModeEx(pCtx)))
4262 {
4263 /* Protected mode checks */
4264 /* CS */
4265 Assert(pCtx->cs.Attr.n.u1Present);
4266 Assert(!(pCtx->cs.Attr.u & 0xf00));
4267 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4268 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4269 || !(pCtx->cs.Attr.n.u1Granularity));
4270 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4271 || (pCtx->cs.Attr.n.u1Granularity));
4272 /* CS cannot be loaded with NULL in protected mode. */
4273 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4274 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4275 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4276 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4277 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4278 else
4279 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4280 /* SS */
4281 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4282 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4283 if ( !(pCtx->cr0 & X86_CR0_PE)
4284 || pCtx->cs.Attr.n.u4Type == 3)
4285 {
4286 Assert(!pCtx->ss.Attr.n.u2Dpl);
4287 }
4288 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4289 {
4290 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4291 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4292 Assert(pCtx->ss.Attr.n.u1Present);
4293 Assert(!(pCtx->ss.Attr.u & 0xf00));
4294 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4295 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4296 || !(pCtx->ss.Attr.n.u1Granularity));
4297 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4298 || (pCtx->ss.Attr.n.u1Granularity));
4299 }
4300 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4301 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4302 {
4303 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4304 Assert(pCtx->ds.Attr.n.u1Present);
4305 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4306 Assert(!(pCtx->ds.Attr.u & 0xf00));
4307 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->ds.Attr.n.u1Granularity));
4310 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4311 || (pCtx->ds.Attr.n.u1Granularity));
4312 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4313 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4314 }
4315 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4316 {
4317 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4318 Assert(pCtx->es.Attr.n.u1Present);
4319 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4320 Assert(!(pCtx->es.Attr.u & 0xf00));
4321 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4322 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4323 || !(pCtx->es.Attr.n.u1Granularity));
4324 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4325 || (pCtx->es.Attr.n.u1Granularity));
4326 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4327 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4328 }
4329 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4330 {
4331 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4332 Assert(pCtx->fs.Attr.n.u1Present);
4333 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4334 Assert(!(pCtx->fs.Attr.u & 0xf00));
4335 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4336 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4337 || !(pCtx->fs.Attr.n.u1Granularity));
4338 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4339 || (pCtx->fs.Attr.n.u1Granularity));
4340 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4341 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4342 }
4343 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4344 {
4345 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4346 Assert(pCtx->gs.Attr.n.u1Present);
4347 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4348 Assert(!(pCtx->gs.Attr.u & 0xf00));
4349 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4350 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4351 || !(pCtx->gs.Attr.n.u1Granularity));
4352 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4353 || (pCtx->gs.Attr.n.u1Granularity));
4354 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4355 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4356 }
4357 /* 64-bit capable CPUs. */
4358# if HC_ARCH_BITS == 64
4359 Assert(!(pCtx->cs.u64Base >> 32));
4360 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4361 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4362 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4363# endif
4364 }
4365 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4366 || ( CPUMIsGuestInRealModeEx(pCtx)
4367 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4368 {
4369 /* Real and v86 mode checks. */
4370 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4371 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4372 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4373 {
4374 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4375 }
4376 else
4377 {
4378 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4379 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4380 }
4381
4382 /* CS */
4383 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4384 Assert(pCtx->cs.u32Limit == 0xffff);
4385 Assert(u32CSAttr == 0xf3);
4386 /* SS */
4387 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4388 Assert(pCtx->ss.u32Limit == 0xffff);
4389 Assert(u32SSAttr == 0xf3);
4390 /* DS */
4391 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4392 Assert(pCtx->ds.u32Limit == 0xffff);
4393 Assert(u32DSAttr == 0xf3);
4394 /* ES */
4395 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4396 Assert(pCtx->es.u32Limit == 0xffff);
4397 Assert(u32ESAttr == 0xf3);
4398 /* FS */
4399 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4400 Assert(pCtx->fs.u32Limit == 0xffff);
4401 Assert(u32FSAttr == 0xf3);
4402 /* GS */
4403 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4404 Assert(pCtx->gs.u32Limit == 0xffff);
4405 Assert(u32GSAttr == 0xf3);
4406 /* 64-bit capable CPUs. */
4407# if HC_ARCH_BITS == 64
4408 Assert(!(pCtx->cs.u64Base >> 32));
4409 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4410 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4411 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu The cross context virtual CPU structure.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4436 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4437 AssertRCReturn(rc, rc);
4438
4439 uint32_t u32Access = pSelReg->Attr.u;
4440 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4441 {
4442 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4443 u32Access = 0xf3;
4444 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4445 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4446 }
4447 else
4448 {
4449 /*
4450 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4451 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4452 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4453 * loaded in protected-mode have their attribute as 0.
4454 */
4455 if (!u32Access)
4456 u32Access = X86DESCATTR_UNUSABLE;
4457 }
4458
4459 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4460 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4461 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4462
4463 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4464 AssertRCReturn(rc, rc);
4465 return rc;
4466}
4467
4468
4469/**
4470 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4471 * into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVCpu The cross context virtual CPU structure.
4475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4476 * out-of-sync. Make sure to update the required fields
4477 * before using them.
4478 *
4479 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4480 * @remarks No-long-jump zone!!!
4481 */
4482static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4483{
4484 int rc = VERR_INTERNAL_ERROR_5;
4485 PVM pVM = pVCpu->CTX_SUFF(pVM);
4486
4487 /*
4488 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4489 */
4490 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4491 {
4492 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4493 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4494 {
4495 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4496 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4501 }
4502
4503#ifdef VBOX_WITH_REM
4504 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4505 {
4506 Assert(pVM->hm.s.vmx.pRealModeTSS);
4507 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4508 if ( pVCpu->hm.s.vmx.fWasInRealMode
4509 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4510 {
4511 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4512 in real-mode (e.g. OpenBSD 4.0) */
4513 REMFlushTBs(pVM);
4514 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4515 pVCpu->hm.s.vmx.fWasInRealMode = false;
4516 }
4517 }
4518#endif
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4520 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4523 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4526 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4529 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4532 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4535 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4536 AssertRCReturn(rc, rc);
4537
4538#ifdef VBOX_STRICT
4539 /* Validate. */
4540 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4541#endif
4542
4543 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4544 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4545 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4546 }
4547
4548 /*
4549 * Guest TR.
4550 */
4551 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4552 {
4553 /*
4554 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4555 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4556 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4557 */
4558 uint16_t u16Sel = 0;
4559 uint32_t u32Limit = 0;
4560 uint64_t u64Base = 0;
4561 uint32_t u32AccessRights = 0;
4562
4563 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4564 {
4565 u16Sel = pMixedCtx->tr.Sel;
4566 u32Limit = pMixedCtx->tr.u32Limit;
4567 u64Base = pMixedCtx->tr.u64Base;
4568 u32AccessRights = pMixedCtx->tr.Attr.u;
4569 }
4570 else
4571 {
4572 Assert(pVM->hm.s.vmx.pRealModeTSS);
4573 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4574
4575 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4576 RTGCPHYS GCPhys;
4577 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4578 AssertRCReturn(rc, rc);
4579
4580 X86DESCATTR DescAttr;
4581 DescAttr.u = 0;
4582 DescAttr.n.u1Present = 1;
4583 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4584
4585 u16Sel = 0;
4586 u32Limit = HM_VTX_TSS_SIZE;
4587 u64Base = GCPhys; /* in real-mode phys = virt. */
4588 u32AccessRights = DescAttr.u;
4589 }
4590
4591 /* Validate. */
4592 Assert(!(u16Sel & RT_BIT(2)));
4593 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4594 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4595 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4596 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4597 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4598 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4599 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4600 Assert( (u32Limit & 0xfff) == 0xfff
4601 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4602 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4603 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4604
4605 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4606 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4607 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4609 AssertRCReturn(rc, rc);
4610
4611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4613 }
4614
4615 /*
4616 * Guest GDTR.
4617 */
4618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4621 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4622 AssertRCReturn(rc, rc);
4623
4624 /* Validate. */
4625 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4626
4627 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4628 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4629 }
4630
4631 /*
4632 * Guest LDTR.
4633 */
4634 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4635 {
4636 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4637 uint32_t u32Access = 0;
4638 if (!pMixedCtx->ldtr.Attr.u)
4639 u32Access = X86DESCATTR_UNUSABLE;
4640 else
4641 u32Access = pMixedCtx->ldtr.Attr.u;
4642
4643 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4647 AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4674 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4675 AssertRCReturn(rc, rc);
4676
4677 /* Validate. */
4678 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4679
4680 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4681 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4682 }
4683
4684 return VINF_SUCCESS;
4685}
4686
4687
4688/**
4689 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4690 * areas.
4691 *
4692 * These MSRs will automatically be loaded to the host CPU on every successful
4693 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4694 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4695 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4696 *
4697 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4698 *
4699 * @returns VBox status code.
4700 * @param pVCpu The cross context virtual CPU structure.
4701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4702 * out-of-sync. Make sure to update the required fields
4703 * before using them.
4704 *
4705 * @remarks No-long-jump zone!!!
4706 */
4707static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4708{
4709 AssertPtr(pVCpu);
4710 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4711
4712 /*
4713 * MSRs that we use the auto-load/store MSR area in the VMCS.
4714 */
4715 PVM pVM = pVCpu->CTX_SUFF(pVM);
4716 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4717 {
4718 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4719#if HC_ARCH_BITS == 32
4720 if (pVM->hm.s.fAllow64BitGuests)
4721 {
4722 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4723 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4724 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4726 AssertRCReturn(rc, rc);
4727# ifdef LOG_ENABLED
4728 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4729 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4730 {
4731 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4732 pMsr->u64Value));
4733 }
4734# endif
4735 }
4736#endif
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738 }
4739
4740 /*
4741 * Guest Sysenter MSRs.
4742 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743 * VM-exits on WRMSRs for these MSRs.
4744 */
4745 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4746 {
4747 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4748 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4749 }
4750
4751 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4752 {
4753 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4754 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4755 }
4756
4757 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4758 {
4759 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4760 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4761 }
4762
4763 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4764 {
4765 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4766 {
4767 /*
4768 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4769 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4770 */
4771 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4772 {
4773 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4774 AssertRCReturn(rc,rc);
4775 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4776 }
4777 else
4778 {
4779 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4780 NULL /* pfAddedAndUpdated */);
4781 AssertRCReturn(rc, rc);
4782
4783 /* We need to intercept reads too, see @bugref{7386#c16}. */
4784 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4785 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4786 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4787 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4788 }
4789 }
4790 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4791 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4792 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4793 }
4794
4795 return VINF_SUCCESS;
4796}
4797
4798
4799/**
4800 * Loads the guest activity state into the guest-state area in the VMCS.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu The cross context virtual CPU structure.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 *
4808 * @remarks No-long-jump zone!!!
4809 */
4810static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811{
4812 NOREF(pMixedCtx);
4813 /** @todo See if we can make use of other states, e.g.
4814 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4815 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4816 {
4817 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4818 AssertRCReturn(rc, rc);
4819
4820 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4821 }
4822 return VINF_SUCCESS;
4823}
4824
4825
4826#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4827/**
4828 * Check if guest state allows safe use of 32-bit switcher again.
4829 *
4830 * Segment bases and protected mode structures must be 32-bit addressable
4831 * because the 32-bit switcher will ignore high dword when writing these VMCS
4832 * fields. See @bugref{8432} for details.
4833 *
4834 * @returns true if safe, false if must continue to use the 64-bit switcher.
4835 * @param pVCpu The cross context virtual CPU structure.
4836 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4837 * out-of-sync. Make sure to update the required fields
4838 * before using them.
4839 *
4840 * @remarks No-long-jump zone!!!
4841 */
4842static bool hmR0VmxIs32BitSwitcherSafe(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4843{
4844 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4845 return false;
4846 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4847 return false;
4848 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4849 return false;
4850 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4851 return false;
4852 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4853 return false;
4854 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4855 return false;
4856 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4857 return false;
4858 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4859 return false;
4860 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4861 return false;
4862 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4863 return false;
4864 /* All good, bases are 32-bit. */
4865 return true;
4866}
4867#endif
4868
4869
4870/**
4871 * Sets up the appropriate function to run guest code.
4872 *
4873 * @returns VBox status code.
4874 * @param pVCpu The cross context virtual CPU structure.
4875 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4876 * out-of-sync. Make sure to update the required fields
4877 * before using them.
4878 *
4879 * @remarks No-long-jump zone!!!
4880 */
4881static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4882{
4883 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4884 {
4885#ifndef VBOX_ENABLE_64_BITS_GUESTS
4886 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4887#endif
4888 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4889#if HC_ARCH_BITS == 32
4890 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4891 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4892 {
4893 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4894 {
4895 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4896 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4897 | HM_CHANGED_VMX_ENTRY_CTLS
4898 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4899 }
4900 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4901
4902 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4903 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4904 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4905 }
4906#else
4907 /* 64-bit host. */
4908 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4909#endif
4910 }
4911 else
4912 {
4913 /* Guest is not in long mode, use the 32-bit handler. */
4914#if HC_ARCH_BITS == 32
4915 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4916 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4917 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4918 {
4919 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4920 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4921 | HM_CHANGED_VMX_ENTRY_CTLS
4922 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4923 }
4924# ifdef VBOX_ENABLE_64_BITS_GUESTS
4925 /* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design. See @bugref{8432#c7}.
4926 * Except if Real-on-V86 is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4927 * state where it's safe to use the 32-bit switcher again.
4928 */
4929 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4930 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4931
4932 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4933 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4934 else
4935 {
4936 Assert(!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active);
4937 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4938 if (hmR0VmxIs32BitSwitcherSafe(pVCpu, pMixedCtx))
4939 {
4940 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4941 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4942 }
4943 }
4944# else
4945 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4946# endif
4947#else
4948 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4949#endif
4950 }
4951 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4952 return VINF_SUCCESS;
4953}
4954
4955
4956/**
4957 * Wrapper for running the guest code in VT-x.
4958 *
4959 * @returns VBox status code, no informational status codes.
4960 * @param pVM The cross context VM structure.
4961 * @param pVCpu The cross context virtual CPU structure.
4962 * @param pCtx Pointer to the guest-CPU context.
4963 *
4964 * @remarks No-long-jump zone!!!
4965 */
4966DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4967{
4968 /*
4969 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4970 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4971 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4972 */
4973 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4974 /** @todo Add stats for resume vs launch. */
4975#ifdef VBOX_WITH_KERNEL_USING_XMM
4976 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4977#else
4978 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4979#endif
4980 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4981 return rc;
4982}
4983
4984
4985/**
4986 * Reports world-switch error and dumps some useful debug info.
4987 *
4988 * @param pVM The cross context VM structure.
4989 * @param pVCpu The cross context virtual CPU structure.
4990 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4991 * @param pCtx Pointer to the guest-CPU context.
4992 * @param pVmxTransient Pointer to the VMX transient structure (only
4993 * exitReason updated).
4994 */
4995static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4996{
4997 Assert(pVM);
4998 Assert(pVCpu);
4999 Assert(pCtx);
5000 Assert(pVmxTransient);
5001 HMVMX_ASSERT_PREEMPT_SAFE();
5002
5003 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5004 switch (rcVMRun)
5005 {
5006 case VERR_VMX_INVALID_VMXON_PTR:
5007 AssertFailed();
5008 break;
5009 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5010 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5011 {
5012 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5013 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5014 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5015 AssertRC(rc);
5016
5017 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5018 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5019 Cannot do it here as we may have been long preempted. */
5020
5021#ifdef VBOX_STRICT
5022 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5023 pVmxTransient->uExitReason));
5024 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5025 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5026 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5027 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5028 else
5029 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5030 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5031 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5032
5033 /* VMX control bits. */
5034 uint32_t u32Val;
5035 uint64_t u64Val;
5036 RTHCUINTREG uHCReg;
5037 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5038 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5039 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5040 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5041 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5042 {
5043 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5044 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5045 }
5046 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5050 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5051 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5054 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5055 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5056 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5057 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5058 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5059 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5060 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5061 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5062 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5063 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5064 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5065 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5066 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5067 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5068 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5069 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5070 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5071 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5072 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5073 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5074 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5075 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5076 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5077 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5078 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5079 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5080 if (pVM->hm.s.fNestedPaging)
5081 {
5082 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5083 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5084 }
5085
5086 /* Guest bits. */
5087 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5088 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5089 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5090 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5091 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5092 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5093 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5094 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5095
5096 /* Host bits. */
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5098 Log4(("Host CR0 %#RHr\n", uHCReg));
5099 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5100 Log4(("Host CR3 %#RHr\n", uHCReg));
5101 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5102 Log4(("Host CR4 %#RHr\n", uHCReg));
5103
5104 RTGDTR HostGdtr;
5105 PCX86DESCHC pDesc;
5106 ASMGetGDTR(&HostGdtr);
5107 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5108 Log4(("Host CS %#08x\n", u32Val));
5109 if (u32Val < HostGdtr.cbGdt)
5110 {
5111 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5112 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5113 }
5114
5115 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5116 Log4(("Host DS %#08x\n", u32Val));
5117 if (u32Val < HostGdtr.cbGdt)
5118 {
5119 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5120 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5121 }
5122
5123 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5124 Log4(("Host ES %#08x\n", u32Val));
5125 if (u32Val < HostGdtr.cbGdt)
5126 {
5127 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5128 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5129 }
5130
5131 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5132 Log4(("Host FS %#08x\n", u32Val));
5133 if (u32Val < HostGdtr.cbGdt)
5134 {
5135 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5136 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5137 }
5138
5139 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5140 Log4(("Host GS %#08x\n", u32Val));
5141 if (u32Val < HostGdtr.cbGdt)
5142 {
5143 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5144 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5145 }
5146
5147 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5148 Log4(("Host SS %#08x\n", u32Val));
5149 if (u32Val < HostGdtr.cbGdt)
5150 {
5151 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5152 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5153 }
5154
5155 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5156 Log4(("Host TR %#08x\n", u32Val));
5157 if (u32Val < HostGdtr.cbGdt)
5158 {
5159 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5160 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5161 }
5162
5163 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5164 Log4(("Host TR Base %#RHv\n", uHCReg));
5165 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5166 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5167 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5168 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5169 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5170 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5171 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5172 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5173 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5174 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5175 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5176 Log4(("Host RSP %#RHv\n", uHCReg));
5177 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5178 Log4(("Host RIP %#RHv\n", uHCReg));
5179# if HC_ARCH_BITS == 64
5180 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5181 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5182 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5183 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5184 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5185 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5186# endif
5187#endif /* VBOX_STRICT */
5188 break;
5189 }
5190
5191 default:
5192 /* Impossible */
5193 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5194 break;
5195 }
5196 NOREF(pVM); NOREF(pCtx);
5197}
5198
5199
5200#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5201#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5202# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5203#endif
5204#ifdef VBOX_STRICT
5205static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5206{
5207 switch (idxField)
5208 {
5209 case VMX_VMCS_GUEST_RIP:
5210 case VMX_VMCS_GUEST_RSP:
5211 case VMX_VMCS_GUEST_SYSENTER_EIP:
5212 case VMX_VMCS_GUEST_SYSENTER_ESP:
5213 case VMX_VMCS_GUEST_GDTR_BASE:
5214 case VMX_VMCS_GUEST_IDTR_BASE:
5215 case VMX_VMCS_GUEST_CS_BASE:
5216 case VMX_VMCS_GUEST_DS_BASE:
5217 case VMX_VMCS_GUEST_ES_BASE:
5218 case VMX_VMCS_GUEST_FS_BASE:
5219 case VMX_VMCS_GUEST_GS_BASE:
5220 case VMX_VMCS_GUEST_SS_BASE:
5221 case VMX_VMCS_GUEST_LDTR_BASE:
5222 case VMX_VMCS_GUEST_TR_BASE:
5223 case VMX_VMCS_GUEST_CR3:
5224 return true;
5225 }
5226 return false;
5227}
5228
5229static bool hmR0VmxIsValidReadField(uint32_t idxField)
5230{
5231 switch (idxField)
5232 {
5233 /* Read-only fields. */
5234 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5235 return true;
5236 }
5237 /* Remaining readable fields should also be writable. */
5238 return hmR0VmxIsValidWriteField(idxField);
5239}
5240#endif /* VBOX_STRICT */
5241
5242
5243/**
5244 * Executes the specified handler in 64-bit mode.
5245 *
5246 * @returns VBox status code (no informational status codes).
5247 * @param pVM The cross context VM structure.
5248 * @param pVCpu The cross context virtual CPU structure.
5249 * @param pCtx Pointer to the guest CPU context.
5250 * @param enmOp The operation to perform.
5251 * @param cParams Number of parameters.
5252 * @param paParam Array of 32-bit parameters.
5253 */
5254VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5255 uint32_t cParams, uint32_t *paParam)
5256{
5257 NOREF(pCtx);
5258
5259 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5260 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5261 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5262 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5263
5264#ifdef VBOX_STRICT
5265 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5266 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5267
5268 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5269 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5270#endif
5271
5272 /* Disable interrupts. */
5273 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5274
5275#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5276 RTCPUID idHostCpu = RTMpCpuId();
5277 CPUMR0SetLApic(pVCpu, idHostCpu);
5278#endif
5279
5280 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5281 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5282
5283 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5284 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5285 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5286
5287 /* Leave VMX Root Mode. */
5288 VMXDisable();
5289
5290 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5291
5292 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5293 CPUMSetHyperEIP(pVCpu, enmOp);
5294 for (int i = (int)cParams - 1; i >= 0; i--)
5295 CPUMPushHyper(pVCpu, paParam[i]);
5296
5297 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5298
5299 /* Call the switcher. */
5300 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5301 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5302
5303 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5304 /* Make sure the VMX instructions don't cause #UD faults. */
5305 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5306
5307 /* Re-enter VMX Root Mode */
5308 int rc2 = VMXEnable(HCPhysCpuPage);
5309 if (RT_FAILURE(rc2))
5310 {
5311 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5312 ASMSetFlags(fOldEFlags);
5313 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5314 return rc2;
5315 }
5316
5317 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5318 AssertRC(rc2);
5319 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5320 Assert(!(ASMGetFlags() & X86_EFL_IF));
5321 ASMSetFlags(fOldEFlags);
5322 return rc;
5323}
5324
5325
5326/**
5327 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5328 * supporting 64-bit guests.
5329 *
5330 * @returns VBox status code.
5331 * @param fResume Whether to VMLAUNCH or VMRESUME.
5332 * @param pCtx Pointer to the guest-CPU context.
5333 * @param pCache Pointer to the VMCS cache.
5334 * @param pVM The cross context VM structure.
5335 * @param pVCpu The cross context virtual CPU structure.
5336 */
5337DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5338{
5339 NOREF(fResume);
5340
5341 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5342 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5343
5344#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5345 pCache->uPos = 1;
5346 pCache->interPD = PGMGetInterPaeCR3(pVM);
5347 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5348#endif
5349
5350#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5351 pCache->TestIn.HCPhysCpuPage = 0;
5352 pCache->TestIn.HCPhysVmcs = 0;
5353 pCache->TestIn.pCache = 0;
5354 pCache->TestOut.HCPhysVmcs = 0;
5355 pCache->TestOut.pCache = 0;
5356 pCache->TestOut.pCtx = 0;
5357 pCache->TestOut.eflags = 0;
5358#else
5359 NOREF(pCache);
5360#endif
5361
5362 uint32_t aParam[10];
5363 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5364 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5365 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5366 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5367 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5368 aParam[5] = 0;
5369 aParam[6] = VM_RC_ADDR(pVM, pVM);
5370 aParam[7] = 0;
5371 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5372 aParam[9] = 0;
5373
5374#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5375 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5376 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5377#endif
5378 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5379
5380#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5381 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5382 Assert(pCtx->dr[4] == 10);
5383 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5384#endif
5385
5386#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5387 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5388 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5389 pVCpu->hm.s.vmx.HCPhysVmcs));
5390 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5391 pCache->TestOut.HCPhysVmcs));
5392 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5393 pCache->TestOut.pCache));
5394 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5395 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5396 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5397 pCache->TestOut.pCtx));
5398 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5399#endif
5400 return rc;
5401}
5402
5403
5404/**
5405 * Initialize the VMCS-Read cache.
5406 *
5407 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5408 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5409 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5410 * (those that have a 32-bit FULL & HIGH part).
5411 *
5412 * @returns VBox status code.
5413 * @param pVM The cross context VM structure.
5414 * @param pVCpu The cross context virtual CPU structure.
5415 */
5416static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5417{
5418#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5419{ \
5420 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5421 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5422 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5423 ++cReadFields; \
5424}
5425
5426 AssertPtr(pVM);
5427 AssertPtr(pVCpu);
5428 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5429 uint32_t cReadFields = 0;
5430
5431 /*
5432 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5433 * and serve to indicate exceptions to the rules.
5434 */
5435
5436 /* Guest-natural selector base fields. */
5437#if 0
5438 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5441#endif
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5443 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5444 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5445 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5446 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5447 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5454#if 0
5455 /* Unused natural width guest-state fields. */
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5458#endif
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5461
5462 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5463#if 0
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5473#endif
5474
5475 /* Natural width guest-state fields. */
5476 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5477#if 0
5478 /* Currently unused field. */
5479 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5480#endif
5481
5482 if (pVM->hm.s.fNestedPaging)
5483 {
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5485 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5486 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5487 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5488 }
5489 else
5490 {
5491 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5492 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5493 }
5494
5495#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5496 return VINF_SUCCESS;
5497}
5498
5499
5500/**
5501 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5502 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5503 * darwin, running 64-bit guests).
5504 *
5505 * @returns VBox status code.
5506 * @param pVCpu The cross context virtual CPU structure.
5507 * @param idxField The VMCS field encoding.
5508 * @param u64Val 16, 32 or 64-bit value.
5509 */
5510VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5511{
5512 int rc;
5513 switch (idxField)
5514 {
5515 /*
5516 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5517 */
5518 /* 64-bit Control fields. */
5519 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5520 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5521 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5522 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5523 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5524 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5525 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5526 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5527 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5528 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5529 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5530 case VMX_VMCS64_CTRL_EPTP_FULL:
5531 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5532 /* 64-bit Guest-state fields. */
5533 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5534 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5535 case VMX_VMCS64_GUEST_PAT_FULL:
5536 case VMX_VMCS64_GUEST_EFER_FULL:
5537 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5538 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5539 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5540 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5541 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5542 /* 64-bit Host-state fields. */
5543 case VMX_VMCS64_HOST_PAT_FULL:
5544 case VMX_VMCS64_HOST_EFER_FULL:
5545 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5546 {
5547 rc = VMXWriteVmcs32(idxField, u64Val);
5548 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5549 break;
5550 }
5551
5552 /*
5553 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5554 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5555 */
5556 /* Natural-width Guest-state fields. */
5557 case VMX_VMCS_GUEST_CR3:
5558 case VMX_VMCS_GUEST_ES_BASE:
5559 case VMX_VMCS_GUEST_CS_BASE:
5560 case VMX_VMCS_GUEST_SS_BASE:
5561 case VMX_VMCS_GUEST_DS_BASE:
5562 case VMX_VMCS_GUEST_FS_BASE:
5563 case VMX_VMCS_GUEST_GS_BASE:
5564 case VMX_VMCS_GUEST_LDTR_BASE:
5565 case VMX_VMCS_GUEST_TR_BASE:
5566 case VMX_VMCS_GUEST_GDTR_BASE:
5567 case VMX_VMCS_GUEST_IDTR_BASE:
5568 case VMX_VMCS_GUEST_RSP:
5569 case VMX_VMCS_GUEST_RIP:
5570 case VMX_VMCS_GUEST_SYSENTER_ESP:
5571 case VMX_VMCS_GUEST_SYSENTER_EIP:
5572 {
5573 if (!(u64Val >> 32))
5574 {
5575 /* If this field is 64-bit, VT-x will zero out the top bits. */
5576 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5577 }
5578 else
5579 {
5580 /* Assert that only the 32->64 switcher case should ever come here. */
5581 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5582 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5583 }
5584 break;
5585 }
5586
5587 default:
5588 {
5589 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5590 rc = VERR_INVALID_PARAMETER;
5591 break;
5592 }
5593 }
5594 AssertRCReturn(rc, rc);
5595 return rc;
5596}
5597
5598
5599/**
5600 * Queue up a VMWRITE by using the VMCS write cache.
5601 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5602 *
5603 * @param pVCpu The cross context virtual CPU structure.
5604 * @param idxField The VMCS field encoding.
5605 * @param u64Val 16, 32 or 64-bit value.
5606 */
5607VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5608{
5609 AssertPtr(pVCpu);
5610 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5611
5612 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5613 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5614
5615 /* Make sure there are no duplicates. */
5616 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5617 {
5618 if (pCache->Write.aField[i] == idxField)
5619 {
5620 pCache->Write.aFieldVal[i] = u64Val;
5621 return VINF_SUCCESS;
5622 }
5623 }
5624
5625 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5626 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5627 pCache->Write.cValidEntries++;
5628 return VINF_SUCCESS;
5629}
5630#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5631
5632
5633/**
5634 * Sets up the usage of TSC-offsetting and updates the VMCS.
5635 *
5636 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5637 * VMX preemption timer.
5638 *
5639 * @returns VBox status code.
5640 * @param pVM The cross context VM structure.
5641 * @param pVCpu The cross context virtual CPU structure.
5642 *
5643 * @remarks No-long-jump zone!!!
5644 */
5645static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5646{
5647 int rc;
5648 bool fOffsettedTsc;
5649 bool fParavirtTsc;
5650 if (pVM->hm.s.vmx.fUsePreemptTimer)
5651 {
5652 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5653 &fOffsettedTsc, &fParavirtTsc);
5654
5655 /* Make sure the returned values have sane upper and lower boundaries. */
5656 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5657 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5658 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5659 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5660
5661 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5662 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5663 }
5664 else
5665 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5666
5667 /** @todo later optimize this to be done elsewhere and not before every
5668 * VM-entry. */
5669 if (fParavirtTsc)
5670 {
5671 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5672 information before every VM-entry, hence disable it for performance sake. */
5673#if 0
5674 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5675 AssertRC(rc);
5676#endif
5677 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5678 }
5679
5680 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5681 {
5682 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5683 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5684
5685 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5686 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5687 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5688 }
5689 else
5690 {
5691 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5692 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5693 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5694 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5695 }
5696}
5697
5698
5699/**
5700 * Determines if an exception is a contributory exception.
5701 *
5702 * Contributory exceptions are ones which can cause double-faults unless the
5703 * original exception was a benign exception. Page-fault is intentionally not
5704 * included here as it's a conditional contributory exception.
5705 *
5706 * @returns true if the exception is contributory, false otherwise.
5707 * @param uVector The exception vector.
5708 */
5709DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5710{
5711 switch (uVector)
5712 {
5713 case X86_XCPT_GP:
5714 case X86_XCPT_SS:
5715 case X86_XCPT_NP:
5716 case X86_XCPT_TS:
5717 case X86_XCPT_DE:
5718 return true;
5719 default:
5720 break;
5721 }
5722 return false;
5723}
5724
5725
5726/**
5727 * Sets an event as a pending event to be injected into the guest.
5728 *
5729 * @param pVCpu The cross context virtual CPU structure.
5730 * @param u32IntInfo The VM-entry interruption-information field.
5731 * @param cbInstr The VM-entry instruction length in bytes (for software
5732 * interrupts, exceptions and privileged software
5733 * exceptions).
5734 * @param u32ErrCode The VM-entry exception error code.
5735 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5736 * page-fault.
5737 *
5738 * @remarks Statistics counter assumes this is a guest event being injected or
5739 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5740 * always incremented.
5741 */
5742DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5743 RTGCUINTPTR GCPtrFaultAddress)
5744{
5745 Assert(!pVCpu->hm.s.Event.fPending);
5746 pVCpu->hm.s.Event.fPending = true;
5747 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5748 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5749 pVCpu->hm.s.Event.cbInstr = cbInstr;
5750 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5751}
5752
5753
5754/**
5755 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5756 *
5757 * @param pVCpu The cross context virtual CPU structure.
5758 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5759 * out-of-sync. Make sure to update the required fields
5760 * before using them.
5761 */
5762DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5763{
5764 NOREF(pMixedCtx);
5765 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5766 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5767 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5768 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5769}
5770
5771
5772/**
5773 * Handle a condition that occurred while delivering an event through the guest
5774 * IDT.
5775 *
5776 * @returns Strict VBox status code (i.e. informational status codes too).
5777 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5778 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5779 * to continue execution of the guest which will delivery the \#DF.
5780 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5781 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5782 *
5783 * @param pVCpu The cross context virtual CPU structure.
5784 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5785 * out-of-sync. Make sure to update the required fields
5786 * before using them.
5787 * @param pVmxTransient Pointer to the VMX transient structure.
5788 *
5789 * @remarks No-long-jump zone!!!
5790 */
5791static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5792{
5793 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5794
5795 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5796 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5797
5798 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5799 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5800 {
5801 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5802 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5803
5804 typedef enum
5805 {
5806 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5807 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5808 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5809 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5810 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5811 } VMXREFLECTXCPT;
5812
5813 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5814 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5815 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5816 {
5817 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5818 {
5819 enmReflect = VMXREFLECTXCPT_XCPT;
5820#ifdef VBOX_STRICT
5821 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5822 && uExitVector == X86_XCPT_PF)
5823 {
5824 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5825 }
5826#endif
5827 if ( uExitVector == X86_XCPT_PF
5828 && uIdtVector == X86_XCPT_PF)
5829 {
5830 pVmxTransient->fVectoringDoublePF = true;
5831 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5832 }
5833 else if ( uExitVector == X86_XCPT_AC
5834 && uIdtVector == X86_XCPT_AC)
5835 {
5836 enmReflect = VMXREFLECTXCPT_HANG;
5837 Log4(("IDT: Nested #AC - Bad guest\n"));
5838 }
5839 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5840 && hmR0VmxIsContributoryXcpt(uExitVector)
5841 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5842 || uIdtVector == X86_XCPT_PF))
5843 {
5844 enmReflect = VMXREFLECTXCPT_DF;
5845 }
5846 else if (uIdtVector == X86_XCPT_DF)
5847 enmReflect = VMXREFLECTXCPT_TF;
5848 }
5849 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5850 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5851 {
5852 /*
5853 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5854 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5855 */
5856 enmReflect = VMXREFLECTXCPT_XCPT;
5857
5858 if (uExitVector == X86_XCPT_PF)
5859 {
5860 pVmxTransient->fVectoringPF = true;
5861 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5862 }
5863 }
5864 }
5865 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5866 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5867 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5868 {
5869 /*
5870 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5871 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5872 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5873 */
5874 enmReflect = VMXREFLECTXCPT_XCPT;
5875 }
5876
5877 /*
5878 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5879 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5880 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5881 *
5882 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5883 */
5884 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5885 && enmReflect == VMXREFLECTXCPT_XCPT
5886 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5887 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5888 {
5889 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5890 }
5891
5892 switch (enmReflect)
5893 {
5894 case VMXREFLECTXCPT_XCPT:
5895 {
5896 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5897 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5898 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5899
5900 uint32_t u32ErrCode = 0;
5901 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5902 {
5903 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5904 AssertRCReturn(rc2, rc2);
5905 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5906 }
5907
5908 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5909 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5910 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5911 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5912 rcStrict = VINF_SUCCESS;
5913 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5914 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5915
5916 break;
5917 }
5918
5919 case VMXREFLECTXCPT_DF:
5920 {
5921 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5922 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5923 rcStrict = VINF_HM_DOUBLE_FAULT;
5924 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5925 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5926
5927 break;
5928 }
5929
5930 case VMXREFLECTXCPT_TF:
5931 {
5932 rcStrict = VINF_EM_RESET;
5933 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5934 uExitVector));
5935 break;
5936 }
5937
5938 case VMXREFLECTXCPT_HANG:
5939 {
5940 rcStrict = VERR_EM_GUEST_CPU_HANG;
5941 break;
5942 }
5943
5944 default:
5945 Assert(rcStrict == VINF_SUCCESS);
5946 break;
5947 }
5948 }
5949 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5950 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5951 && uExitVector != X86_XCPT_DF
5952 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5953 {
5954 /*
5955 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5956 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5957 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5958 */
5959 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5960 {
5961 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5962 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5963 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5964 }
5965 }
5966
5967 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5968 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5969 return rcStrict;
5970}
5971
5972
5973/**
5974 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5975 *
5976 * @returns VBox status code.
5977 * @param pVCpu The cross context virtual CPU structure.
5978 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5979 * out-of-sync. Make sure to update the required fields
5980 * before using them.
5981 *
5982 * @remarks No-long-jump zone!!!
5983 */
5984static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5985{
5986 NOREF(pMixedCtx);
5987
5988 /*
5989 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5990 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5991 */
5992 VMMRZCallRing3Disable(pVCpu);
5993 HM_DISABLE_PREEMPT();
5994
5995 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5996 {
5997 uint32_t uVal = 0;
5998 uint32_t uShadow = 0;
5999 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6000 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6001 AssertRCReturn(rc, rc);
6002
6003 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6004 CPUMSetGuestCR0(pVCpu, uVal);
6005 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6006 }
6007
6008 HM_RESTORE_PREEMPT();
6009 VMMRZCallRing3Enable(pVCpu);
6010 return VINF_SUCCESS;
6011}
6012
6013
6014/**
6015 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6016 *
6017 * @returns VBox status code.
6018 * @param pVCpu The cross context virtual CPU structure.
6019 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6020 * out-of-sync. Make sure to update the required fields
6021 * before using them.
6022 *
6023 * @remarks No-long-jump zone!!!
6024 */
6025static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6026{
6027 NOREF(pMixedCtx);
6028
6029 int rc = VINF_SUCCESS;
6030 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6031 {
6032 uint32_t uVal = 0;
6033 uint32_t uShadow = 0;
6034 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6035 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6036 AssertRCReturn(rc, rc);
6037
6038 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6039 CPUMSetGuestCR4(pVCpu, uVal);
6040 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6041 }
6042 return rc;
6043}
6044
6045
6046/**
6047 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6048 *
6049 * @returns VBox status code.
6050 * @param pVCpu The cross context virtual CPU structure.
6051 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6052 * out-of-sync. Make sure to update the required fields
6053 * before using them.
6054 *
6055 * @remarks No-long-jump zone!!!
6056 */
6057static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6058{
6059 int rc = VINF_SUCCESS;
6060 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6061 {
6062 uint64_t u64Val = 0;
6063 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6064 AssertRCReturn(rc, rc);
6065
6066 pMixedCtx->rip = u64Val;
6067 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6068 }
6069 return rc;
6070}
6071
6072
6073/**
6074 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6075 *
6076 * @returns VBox status code.
6077 * @param pVCpu The cross context virtual CPU structure.
6078 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6079 * out-of-sync. Make sure to update the required fields
6080 * before using them.
6081 *
6082 * @remarks No-long-jump zone!!!
6083 */
6084static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6085{
6086 int rc = VINF_SUCCESS;
6087 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6088 {
6089 uint64_t u64Val = 0;
6090 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6091 AssertRCReturn(rc, rc);
6092
6093 pMixedCtx->rsp = u64Val;
6094 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6095 }
6096 return rc;
6097}
6098
6099
6100/**
6101 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6102 *
6103 * @returns VBox status code.
6104 * @param pVCpu The cross context virtual CPU structure.
6105 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6106 * out-of-sync. Make sure to update the required fields
6107 * before using them.
6108 *
6109 * @remarks No-long-jump zone!!!
6110 */
6111static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6112{
6113 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6114 {
6115 uint32_t uVal = 0;
6116 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6117 AssertRCReturn(rc, rc);
6118
6119 pMixedCtx->eflags.u32 = uVal;
6120 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6121 {
6122 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6123 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6124
6125 pMixedCtx->eflags.Bits.u1VM = 0;
6126 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6127 }
6128
6129 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6130 }
6131 return VINF_SUCCESS;
6132}
6133
6134
6135/**
6136 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6137 * guest-CPU context.
6138 */
6139DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6140{
6141 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6142 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6143 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6144 return rc;
6145}
6146
6147
6148/**
6149 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6150 * from the guest-state area in the VMCS.
6151 *
6152 * @param pVCpu The cross context virtual CPU structure.
6153 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6154 * out-of-sync. Make sure to update the required fields
6155 * before using them.
6156 *
6157 * @remarks No-long-jump zone!!!
6158 */
6159static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6160{
6161 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6162 {
6163 uint32_t uIntrState = 0;
6164 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6165 AssertRC(rc);
6166
6167 if (!uIntrState)
6168 {
6169 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6170 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6171
6172 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6173 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6174 }
6175 else
6176 {
6177 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6178 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6179 {
6180 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6181 AssertRC(rc);
6182 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6183 AssertRC(rc);
6184
6185 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6186 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6187 }
6188 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6189 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6190
6191 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6192 {
6193 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6194 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6195 }
6196 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6197 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6198 }
6199
6200 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6201 }
6202}
6203
6204
6205/**
6206 * Saves the guest's activity state.
6207 *
6208 * @returns VBox status code.
6209 * @param pVCpu The cross context virtual CPU structure.
6210 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6211 * out-of-sync. Make sure to update the required fields
6212 * before using them.
6213 *
6214 * @remarks No-long-jump zone!!!
6215 */
6216static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6217{
6218 NOREF(pMixedCtx);
6219 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6220 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6221 return VINF_SUCCESS;
6222}
6223
6224
6225/**
6226 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6227 * the current VMCS into the guest-CPU context.
6228 *
6229 * @returns VBox status code.
6230 * @param pVCpu The cross context virtual CPU structure.
6231 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6232 * out-of-sync. Make sure to update the required fields
6233 * before using them.
6234 *
6235 * @remarks No-long-jump zone!!!
6236 */
6237static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6238{
6239 int rc = VINF_SUCCESS;
6240 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6241 {
6242 uint32_t u32Val = 0;
6243 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6244 pMixedCtx->SysEnter.cs = u32Val;
6245 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6246 }
6247
6248 uint64_t u64Val = 0;
6249 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6250 {
6251 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6252 pMixedCtx->SysEnter.eip = u64Val;
6253 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6254 }
6255 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6256 {
6257 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6258 pMixedCtx->SysEnter.esp = u64Val;
6259 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6260 }
6261 return rc;
6262}
6263
6264
6265/**
6266 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6267 * the CPU back into the guest-CPU context.
6268 *
6269 * @returns VBox status code.
6270 * @param pVCpu The cross context virtual CPU structure.
6271 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6272 * out-of-sync. Make sure to update the required fields
6273 * before using them.
6274 *
6275 * @remarks No-long-jump zone!!!
6276 */
6277static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6278{
6279 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6280 VMMRZCallRing3Disable(pVCpu);
6281 HM_DISABLE_PREEMPT();
6282
6283 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6284 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6285 {
6286 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6287 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6288 }
6289
6290 HM_RESTORE_PREEMPT();
6291 VMMRZCallRing3Enable(pVCpu);
6292
6293 return VINF_SUCCESS;
6294}
6295
6296
6297/**
6298 * Saves the auto load/store'd guest MSRs from the current VMCS into
6299 * the guest-CPU context.
6300 *
6301 * @returns VBox status code.
6302 * @param pVCpu The cross context virtual CPU structure.
6303 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6304 * out-of-sync. Make sure to update the required fields
6305 * before using them.
6306 *
6307 * @remarks No-long-jump zone!!!
6308 */
6309static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6310{
6311 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6312 return VINF_SUCCESS;
6313
6314 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6315 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6316 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6317 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6318 {
6319 switch (pMsr->u32Msr)
6320 {
6321 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6322 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6323 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6324 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6325 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6326 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6327 break;
6328
6329 default:
6330 {
6331 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6332 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6333 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6334 }
6335 }
6336 }
6337
6338 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6339 return VINF_SUCCESS;
6340}
6341
6342
6343/**
6344 * Saves the guest control registers from the current VMCS into the guest-CPU
6345 * context.
6346 *
6347 * @returns VBox status code.
6348 * @param pVCpu The cross context virtual CPU structure.
6349 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6350 * out-of-sync. Make sure to update the required fields
6351 * before using them.
6352 *
6353 * @remarks No-long-jump zone!!!
6354 */
6355static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6356{
6357 /* Guest CR0. Guest FPU. */
6358 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6359 AssertRCReturn(rc, rc);
6360
6361 /* Guest CR4. */
6362 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6363 AssertRCReturn(rc, rc);
6364
6365 /* Guest CR2 - updated always during the world-switch or in #PF. */
6366 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6367 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6368 {
6369 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6370 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6371
6372 PVM pVM = pVCpu->CTX_SUFF(pVM);
6373 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6374 || ( pVM->hm.s.fNestedPaging
6375 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6376 {
6377 uint64_t u64Val = 0;
6378 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6379 if (pMixedCtx->cr3 != u64Val)
6380 {
6381 CPUMSetGuestCR3(pVCpu, u64Val);
6382 if (VMMRZCallRing3IsEnabled(pVCpu))
6383 {
6384 PGMUpdateCR3(pVCpu, u64Val);
6385 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6386 }
6387 else
6388 {
6389 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6390 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6391 }
6392 }
6393
6394 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6395 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6396 {
6397 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6398 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6399 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6400 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6401 AssertRCReturn(rc, rc);
6402
6403 if (VMMRZCallRing3IsEnabled(pVCpu))
6404 {
6405 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6406 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6407 }
6408 else
6409 {
6410 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6411 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6412 }
6413 }
6414 }
6415
6416 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6417 }
6418
6419 /*
6420 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6421 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6422 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6423 *
6424 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6425 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6426 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6427 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6428 *
6429 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6430 */
6431 if (VMMRZCallRing3IsEnabled(pVCpu))
6432 {
6433 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6434 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6435
6436 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6437 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6438
6439 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6440 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6441 }
6442
6443 return rc;
6444}
6445
6446
6447/**
6448 * Reads a guest segment register from the current VMCS into the guest-CPU
6449 * context.
6450 *
6451 * @returns VBox status code.
6452 * @param pVCpu The cross context virtual CPU structure.
6453 * @param idxSel Index of the selector in the VMCS.
6454 * @param idxLimit Index of the segment limit in the VMCS.
6455 * @param idxBase Index of the segment base in the VMCS.
6456 * @param idxAccess Index of the access rights of the segment in the VMCS.
6457 * @param pSelReg Pointer to the segment selector.
6458 *
6459 * @remarks No-long-jump zone!!!
6460 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6461 * macro as that takes care of whether to read from the VMCS cache or
6462 * not.
6463 */
6464DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6465 PCPUMSELREG pSelReg)
6466{
6467 NOREF(pVCpu);
6468
6469 uint32_t u32Val = 0;
6470 int rc = VMXReadVmcs32(idxSel, &u32Val);
6471 AssertRCReturn(rc, rc);
6472 pSelReg->Sel = (uint16_t)u32Val;
6473 pSelReg->ValidSel = (uint16_t)u32Val;
6474 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6475
6476 rc = VMXReadVmcs32(idxLimit, &u32Val);
6477 AssertRCReturn(rc, rc);
6478 pSelReg->u32Limit = u32Val;
6479
6480 uint64_t u64Val = 0;
6481 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6482 AssertRCReturn(rc, rc);
6483 pSelReg->u64Base = u64Val;
6484
6485 rc = VMXReadVmcs32(idxAccess, &u32Val);
6486 AssertRCReturn(rc, rc);
6487 pSelReg->Attr.u = u32Val;
6488
6489 /*
6490 * If VT-x marks the segment as unusable, most other bits remain undefined:
6491 * - For CS the L, D and G bits have meaning.
6492 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6493 * - For the remaining data segments no bits are defined.
6494 *
6495 * The present bit and the unusable bit has been observed to be set at the
6496 * same time (the selector was supposed to be invalid as we started executing
6497 * a V8086 interrupt in ring-0).
6498 *
6499 * What should be important for the rest of the VBox code, is that the P bit is
6500 * cleared. Some of the other VBox code recognizes the unusable bit, but
6501 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6502 * safe side here, we'll strip off P and other bits we don't care about. If
6503 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6504 *
6505 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6506 */
6507 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6508 {
6509 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6510
6511 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6512 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6513 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6514
6515 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6516#ifdef DEBUG_bird
6517 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6518 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6519 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6520#endif
6521 }
6522 return VINF_SUCCESS;
6523}
6524
6525
6526#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6527# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6528 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6529 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6530#else
6531# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6532 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6533 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6534#endif
6535
6536
6537/**
6538 * Saves the guest segment registers from the current VMCS into the guest-CPU
6539 * context.
6540 *
6541 * @returns VBox status code.
6542 * @param pVCpu The cross context virtual CPU structure.
6543 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6544 * out-of-sync. Make sure to update the required fields
6545 * before using them.
6546 *
6547 * @remarks No-long-jump zone!!!
6548 */
6549static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6550{
6551 /* Guest segment registers. */
6552 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6553 {
6554 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6555 AssertRCReturn(rc, rc);
6556
6557 rc = VMXLOCAL_READ_SEG(CS, cs);
6558 rc |= VMXLOCAL_READ_SEG(SS, ss);
6559 rc |= VMXLOCAL_READ_SEG(DS, ds);
6560 rc |= VMXLOCAL_READ_SEG(ES, es);
6561 rc |= VMXLOCAL_READ_SEG(FS, fs);
6562 rc |= VMXLOCAL_READ_SEG(GS, gs);
6563 AssertRCReturn(rc, rc);
6564
6565 /* Restore segment attributes for real-on-v86 mode hack. */
6566 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6567 {
6568 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6569 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6570 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6571 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6572 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6573 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6574 }
6575 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6576 }
6577
6578 return VINF_SUCCESS;
6579}
6580
6581
6582/**
6583 * Saves the guest descriptor table registers and task register from the current
6584 * VMCS into the guest-CPU context.
6585 *
6586 * @returns VBox status code.
6587 * @param pVCpu The cross context virtual CPU structure.
6588 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6589 * out-of-sync. Make sure to update the required fields
6590 * before using them.
6591 *
6592 * @remarks No-long-jump zone!!!
6593 */
6594static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6595{
6596 int rc = VINF_SUCCESS;
6597
6598 /* Guest LDTR. */
6599 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6600 {
6601 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6602 AssertRCReturn(rc, rc);
6603 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6604 }
6605
6606 /* Guest GDTR. */
6607 uint64_t u64Val = 0;
6608 uint32_t u32Val = 0;
6609 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6610 {
6611 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6612 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6613 pMixedCtx->gdtr.pGdt = u64Val;
6614 pMixedCtx->gdtr.cbGdt = u32Val;
6615 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6616 }
6617
6618 /* Guest IDTR. */
6619 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6620 {
6621 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6622 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6623 pMixedCtx->idtr.pIdt = u64Val;
6624 pMixedCtx->idtr.cbIdt = u32Val;
6625 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6626 }
6627
6628 /* Guest TR. */
6629 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6630 {
6631 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6632 AssertRCReturn(rc, rc);
6633
6634 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6635 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6636 {
6637 rc = VMXLOCAL_READ_SEG(TR, tr);
6638 AssertRCReturn(rc, rc);
6639 }
6640 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6641 }
6642 return rc;
6643}
6644
6645#undef VMXLOCAL_READ_SEG
6646
6647
6648/**
6649 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6650 * context.
6651 *
6652 * @returns VBox status code.
6653 * @param pVCpu The cross context virtual CPU structure.
6654 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6655 * out-of-sync. Make sure to update the required fields
6656 * before using them.
6657 *
6658 * @remarks No-long-jump zone!!!
6659 */
6660static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6661{
6662 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6663 {
6664 if (!pVCpu->hm.s.fUsingHyperDR7)
6665 {
6666 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6667 uint32_t u32Val;
6668 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6669 pMixedCtx->dr[7] = u32Val;
6670 }
6671
6672 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6673 }
6674 return VINF_SUCCESS;
6675}
6676
6677
6678/**
6679 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6680 *
6681 * @returns VBox status code.
6682 * @param pVCpu The cross context virtual CPU structure.
6683 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6684 * out-of-sync. Make sure to update the required fields
6685 * before using them.
6686 *
6687 * @remarks No-long-jump zone!!!
6688 */
6689static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6690{
6691 NOREF(pMixedCtx);
6692
6693 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6694 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6695 return VINF_SUCCESS;
6696}
6697
6698
6699/**
6700 * Saves the entire guest state from the currently active VMCS into the
6701 * guest-CPU context.
6702 *
6703 * This essentially VMREADs all guest-data.
6704 *
6705 * @returns VBox status code.
6706 * @param pVCpu The cross context virtual CPU structure.
6707 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6708 * out-of-sync. Make sure to update the required fields
6709 * before using them.
6710 */
6711static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6712{
6713 Assert(pVCpu);
6714 Assert(pMixedCtx);
6715
6716 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6717 return VINF_SUCCESS;
6718
6719 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6720 again on the ring-3 callback path, there is no real need to. */
6721 if (VMMRZCallRing3IsEnabled(pVCpu))
6722 VMMR0LogFlushDisable(pVCpu);
6723 else
6724 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6725 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6726
6727 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6728 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6729
6730 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6731 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6732
6733 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6734 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6735
6736 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6737 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6738
6739 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6740 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6741
6742 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6743 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6744
6745 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6746 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6747
6748 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6749 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6750
6751 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6752 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6753
6754 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6755 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6756
6757 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6758 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6759 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6760
6761 if (VMMRZCallRing3IsEnabled(pVCpu))
6762 VMMR0LogFlushEnable(pVCpu);
6763
6764 return VINF_SUCCESS;
6765}
6766
6767
6768/**
6769 * Saves basic guest registers needed for IEM instruction execution.
6770 *
6771 * @returns VBox status code (OR-able).
6772 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6773 * @param pMixedCtx Pointer to the CPU context of the guest.
6774 * @param fMemory Whether the instruction being executed operates on
6775 * memory or not. Only CR0 is synced up if clear.
6776 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6777 */
6778static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6779{
6780 /*
6781 * We assume all general purpose registers other than RSP are available.
6782 *
6783 * RIP is a must, as it will be incremented or otherwise changed.
6784 *
6785 * RFLAGS are always required to figure the CPL.
6786 *
6787 * RSP isn't always required, however it's a GPR, so frequently required.
6788 *
6789 * SS and CS are the only segment register needed if IEM doesn't do memory
6790 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6791 *
6792 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6793 * be required for memory accesses.
6794 *
6795 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6796 */
6797 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6798 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6799 if (fNeedRsp)
6800 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6801 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6802 if (!fMemory)
6803 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6804 else
6805 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6806 AssertRCReturn(rc, rc);
6807 return rc;
6808}
6809
6810
6811/**
6812 * Ensures that we've got a complete basic guest-context.
6813 *
6814 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6815 * is for the interpreter.
6816 *
6817 * @returns VBox status code.
6818 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6819 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6820 * needing to be synced in.
6821 * @thread EMT(pVCpu)
6822 */
6823VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6824{
6825 /* Note! Since this is only applicable to VT-x, the implementation is placed
6826 in the VT-x part of the sources instead of the generic stuff. */
6827 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6828 {
6829 /* For now, imply that the caller might change everything too. */
6830 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6831 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6832 }
6833 return VINF_SUCCESS;
6834}
6835
6836
6837/**
6838 * Check per-VM and per-VCPU force flag actions that require us to go back to
6839 * ring-3 for one reason or another.
6840 *
6841 * @returns Strict VBox status code (i.e. informational status codes too)
6842 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6843 * ring-3.
6844 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6845 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6846 * interrupts)
6847 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6848 * all EMTs to be in ring-3.
6849 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6850 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6851 * to the EM loop.
6852 *
6853 * @param pVM The cross context VM structure.
6854 * @param pVCpu The cross context virtual CPU structure.
6855 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6856 * out-of-sync. Make sure to update the required fields
6857 * before using them.
6858 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6859 */
6860static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6861{
6862 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6863
6864 /*
6865 * Anything pending? Should be more likely than not if we're doing a good job.
6866 */
6867 if ( !fStepping
6868 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6869 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6870 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6871 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6872 return VINF_SUCCESS;
6873
6874 /* We need the control registers now, make sure the guest-CPU context is updated. */
6875 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6876 AssertRCReturn(rc3, rc3);
6877
6878 /* Pending HM CR3 sync. */
6879 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6880 {
6881 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6882 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6883 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6884 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6885 }
6886
6887 /* Pending HM PAE PDPEs. */
6888 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6889 {
6890 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6891 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6892 }
6893
6894 /* Pending PGM C3 sync. */
6895 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6896 {
6897 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6898 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6899 if (rcStrict2 != VINF_SUCCESS)
6900 {
6901 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6902 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6903 return rcStrict2;
6904 }
6905 }
6906
6907 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6908 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6909 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6910 {
6911 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6912 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6913 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6914 return rc2;
6915 }
6916
6917 /* Pending VM request packets, such as hardware interrupts. */
6918 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6919 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6920 {
6921 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6922 return VINF_EM_PENDING_REQUEST;
6923 }
6924
6925 /* Pending PGM pool flushes. */
6926 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6927 {
6928 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6929 return VINF_PGM_POOL_FLUSH_PENDING;
6930 }
6931
6932 /* Pending DMA requests. */
6933 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6934 {
6935 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6936 return VINF_EM_RAW_TO_R3;
6937 }
6938
6939 return VINF_SUCCESS;
6940}
6941
6942
6943/**
6944 * Converts any TRPM trap into a pending HM event. This is typically used when
6945 * entering from ring-3 (not longjmp returns).
6946 *
6947 * @param pVCpu The cross context virtual CPU structure.
6948 */
6949static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6950{
6951 Assert(TRPMHasTrap(pVCpu));
6952 Assert(!pVCpu->hm.s.Event.fPending);
6953
6954 uint8_t uVector;
6955 TRPMEVENT enmTrpmEvent;
6956 RTGCUINT uErrCode;
6957 RTGCUINTPTR GCPtrFaultAddress;
6958 uint8_t cbInstr;
6959
6960 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6961 AssertRC(rc);
6962
6963 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6964 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6965 if (enmTrpmEvent == TRPM_TRAP)
6966 {
6967 switch (uVector)
6968 {
6969 case X86_XCPT_NMI:
6970 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6971 break;
6972
6973 case X86_XCPT_BP:
6974 case X86_XCPT_OF:
6975 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6976 break;
6977
6978 case X86_XCPT_PF:
6979 case X86_XCPT_DF:
6980 case X86_XCPT_TS:
6981 case X86_XCPT_NP:
6982 case X86_XCPT_SS:
6983 case X86_XCPT_GP:
6984 case X86_XCPT_AC:
6985 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6986 /* no break! */
6987 default:
6988 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6989 break;
6990 }
6991 }
6992 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6993 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6994 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6995 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6996 else
6997 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6998
6999 rc = TRPMResetTrap(pVCpu);
7000 AssertRC(rc);
7001 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7002 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7003
7004 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7005}
7006
7007
7008/**
7009 * Converts the pending HM event into a TRPM trap.
7010 *
7011 * @param pVCpu The cross context virtual CPU structure.
7012 */
7013static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7014{
7015 Assert(pVCpu->hm.s.Event.fPending);
7016
7017 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7018 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7019 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7020 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7021
7022 /* If a trap was already pending, we did something wrong! */
7023 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7024
7025 TRPMEVENT enmTrapType;
7026 switch (uVectorType)
7027 {
7028 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7029 enmTrapType = TRPM_HARDWARE_INT;
7030 break;
7031
7032 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7033 enmTrapType = TRPM_SOFTWARE_INT;
7034 break;
7035
7036 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7037 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7038 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7039 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7040 enmTrapType = TRPM_TRAP;
7041 break;
7042
7043 default:
7044 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7045 enmTrapType = TRPM_32BIT_HACK;
7046 break;
7047 }
7048
7049 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7050
7051 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7052 AssertRC(rc);
7053
7054 if (fErrorCodeValid)
7055 TRPMSetErrorCode(pVCpu, uErrorCode);
7056
7057 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7058 && uVector == X86_XCPT_PF)
7059 {
7060 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7061 }
7062 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7063 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7064 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7065 {
7066 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7067 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7068 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7069 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7070 }
7071
7072 /* Clear any pending events from the VMCS. */
7073 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7074 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7075
7076 /* We're now done converting the pending event. */
7077 pVCpu->hm.s.Event.fPending = false;
7078}
7079
7080
7081/**
7082 * Does the necessary state syncing before returning to ring-3 for any reason
7083 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7084 *
7085 * @returns VBox status code.
7086 * @param pVM The cross context VM structure.
7087 * @param pVCpu The cross context virtual CPU structure.
7088 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7089 * be out-of-sync. Make sure to update the required
7090 * fields before using them.
7091 * @param fSaveGuestState Whether to save the guest state or not.
7092 *
7093 * @remarks No-long-jmp zone!!!
7094 */
7095static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7096{
7097 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7098 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7099
7100 RTCPUID idCpu = RTMpCpuId();
7101 Log4Func(("HostCpuId=%u\n", idCpu));
7102
7103 /*
7104 * !!! IMPORTANT !!!
7105 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7106 */
7107
7108 /* Save the guest state if necessary. */
7109 if ( fSaveGuestState
7110 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7111 {
7112 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7113 AssertRCReturn(rc, rc);
7114 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7115 }
7116
7117 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7118 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7119 {
7120 if (fSaveGuestState)
7121 {
7122 /* We shouldn't reload CR0 without saving it first. */
7123 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7124 AssertRCReturn(rc, rc);
7125 }
7126 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7127 }
7128
7129 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7130#ifdef VBOX_STRICT
7131 if (CPUMIsHyperDebugStateActive(pVCpu))
7132 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7133#endif
7134 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7135 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7136 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7137 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7138
7139#if HC_ARCH_BITS == 64
7140 /* Restore host-state bits that VT-x only restores partially. */
7141 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7142 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7143 {
7144 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7145 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7146 }
7147 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7148#endif
7149
7150 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7151 if (pVCpu->hm.s.vmx.fLazyMsrs)
7152 {
7153 /* We shouldn't reload the guest MSRs without saving it first. */
7154 if (!fSaveGuestState)
7155 {
7156 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7157 AssertRCReturn(rc, rc);
7158 }
7159 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7160 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7161 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7162 }
7163
7164 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7165 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7166
7167 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7168 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7169 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7170 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7171 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7172 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7173 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7174 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7175
7176 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7177
7178 /** @todo This partially defeats the purpose of having preemption hooks.
7179 * The problem is, deregistering the hooks should be moved to a place that
7180 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7181 * context.
7182 */
7183 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7184 {
7185 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7186 AssertRCReturn(rc, rc);
7187
7188 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7189 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7190 }
7191 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7192 NOREF(idCpu);
7193
7194 return VINF_SUCCESS;
7195}
7196
7197
7198/**
7199 * Leaves the VT-x session.
7200 *
7201 * @returns VBox status code.
7202 * @param pVM The cross context VM structure.
7203 * @param pVCpu The cross context virtual CPU structure.
7204 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7205 * out-of-sync. Make sure to update the required fields
7206 * before using them.
7207 *
7208 * @remarks No-long-jmp zone!!!
7209 */
7210DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7211{
7212 HM_DISABLE_PREEMPT();
7213 HMVMX_ASSERT_CPU_SAFE();
7214 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7215 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7216
7217 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7218 and done this from the VMXR0ThreadCtxCallback(). */
7219 if (!pVCpu->hm.s.fLeaveDone)
7220 {
7221 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7222 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7223 pVCpu->hm.s.fLeaveDone = true;
7224 }
7225 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7226
7227 /*
7228 * !!! IMPORTANT !!!
7229 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7230 */
7231
7232 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7233 /** @todo Deregistering here means we need to VMCLEAR always
7234 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7235 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7236 VMMR0ThreadCtxHookDisable(pVCpu);
7237
7238 /* Leave HM context. This takes care of local init (term). */
7239 int rc = HMR0LeaveCpu(pVCpu);
7240
7241 HM_RESTORE_PREEMPT();
7242 return rc;
7243}
7244
7245
7246/**
7247 * Does the necessary state syncing before doing a longjmp to ring-3.
7248 *
7249 * @returns VBox status code.
7250 * @param pVM The cross context VM structure.
7251 * @param pVCpu The cross context virtual CPU structure.
7252 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7253 * out-of-sync. Make sure to update the required fields
7254 * before using them.
7255 *
7256 * @remarks No-long-jmp zone!!!
7257 */
7258DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7259{
7260 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7261}
7262
7263
7264/**
7265 * Take necessary actions before going back to ring-3.
7266 *
7267 * An action requires us to go back to ring-3. This function does the necessary
7268 * steps before we can safely return to ring-3. This is not the same as longjmps
7269 * to ring-3, this is voluntary and prepares the guest so it may continue
7270 * executing outside HM (recompiler/IEM).
7271 *
7272 * @returns VBox status code.
7273 * @param pVM The cross context VM structure.
7274 * @param pVCpu The cross context virtual CPU structure.
7275 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7276 * out-of-sync. Make sure to update the required fields
7277 * before using them.
7278 * @param rcExit The reason for exiting to ring-3. Can be
7279 * VINF_VMM_UNKNOWN_RING3_CALL.
7280 */
7281static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7282{
7283 Assert(pVM);
7284 Assert(pVCpu);
7285 Assert(pMixedCtx);
7286 HMVMX_ASSERT_PREEMPT_SAFE();
7287
7288 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7289 {
7290 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7291 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7292 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7293 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7294 }
7295
7296 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7297 VMMRZCallRing3Disable(pVCpu);
7298 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7299
7300 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7301 if (pVCpu->hm.s.Event.fPending)
7302 {
7303 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7304 Assert(!pVCpu->hm.s.Event.fPending);
7305 }
7306
7307 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7308 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7309
7310 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7311 and if we're injecting an event we should have a TRPM trap pending. */
7312 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7313#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7314 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7315#endif
7316
7317 /* Save guest state and restore host state bits. */
7318 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7319 AssertRCReturn(rc, rc);
7320 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7321 /* Thread-context hooks are unregistered at this point!!! */
7322
7323 /* Sync recompiler state. */
7324 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7325 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7326 | CPUM_CHANGED_LDTR
7327 | CPUM_CHANGED_GDTR
7328 | CPUM_CHANGED_IDTR
7329 | CPUM_CHANGED_TR
7330 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7331 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7332 if ( pVM->hm.s.fNestedPaging
7333 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7334 {
7335 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7336 }
7337
7338 Assert(!pVCpu->hm.s.fClearTrapFlag);
7339
7340 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7341 if (rcExit != VINF_EM_RAW_INTERRUPT)
7342 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7343
7344 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7345
7346 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7347 VMMRZCallRing3RemoveNotification(pVCpu);
7348 VMMRZCallRing3Enable(pVCpu);
7349
7350 return rc;
7351}
7352
7353
7354/**
7355 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7356 * longjump to ring-3 and possibly get preempted.
7357 *
7358 * @returns VBox status code.
7359 * @param pVCpu The cross context virtual CPU structure.
7360 * @param enmOperation The operation causing the ring-3 longjump.
7361 * @param pvUser Opaque pointer to the guest-CPU context. The data
7362 * may be out-of-sync. Make sure to update the required
7363 * fields before using them.
7364 */
7365static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7366{
7367 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7368 {
7369 /*
7370 * !!! IMPORTANT !!!
7371 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7372 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7373 */
7374 VMMRZCallRing3RemoveNotification(pVCpu);
7375 VMMRZCallRing3Disable(pVCpu);
7376 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7377 RTThreadPreemptDisable(&PreemptState);
7378
7379 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7380 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7381
7382#if HC_ARCH_BITS == 64
7383 /* Restore host-state bits that VT-x only restores partially. */
7384 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7385 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7386 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7387 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7388#endif
7389 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7390 if (pVCpu->hm.s.vmx.fLazyMsrs)
7391 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7392
7393 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7394 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7395 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7396 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7397 {
7398 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7399 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7400 }
7401
7402 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7403 VMMR0ThreadCtxHookDisable(pVCpu);
7404 HMR0LeaveCpu(pVCpu);
7405 RTThreadPreemptRestore(&PreemptState);
7406 return VINF_SUCCESS;
7407 }
7408
7409 Assert(pVCpu);
7410 Assert(pvUser);
7411 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7412 HMVMX_ASSERT_PREEMPT_SAFE();
7413
7414 VMMRZCallRing3Disable(pVCpu);
7415 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7416
7417 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7418 enmOperation));
7419
7420 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7421 AssertRCReturn(rc, rc);
7422
7423 VMMRZCallRing3Enable(pVCpu);
7424 return VINF_SUCCESS;
7425}
7426
7427
7428/**
7429 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7430 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7431 *
7432 * @param pVCpu The cross context virtual CPU structure.
7433 */
7434DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7435{
7436 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7437 {
7438 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7439 {
7440 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7441 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7442 AssertRC(rc);
7443 Log4(("Setup interrupt-window exiting\n"));
7444 }
7445 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7446}
7447
7448
7449/**
7450 * Clears the interrupt-window exiting control in the VMCS.
7451 *
7452 * @param pVCpu The cross context virtual CPU structure.
7453 */
7454DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7455{
7456 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7457 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7458 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7459 AssertRC(rc);
7460 Log4(("Cleared interrupt-window exiting\n"));
7461}
7462
7463
7464/**
7465 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7466 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7467 *
7468 * @param pVCpu The cross context virtual CPU structure.
7469 */
7470DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7471{
7472 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7473 {
7474 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7475 {
7476 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7477 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7478 AssertRC(rc);
7479 Log4(("Setup NMI-window exiting\n"));
7480 }
7481 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7482}
7483
7484
7485/**
7486 * Clears the NMI-window exiting control in the VMCS.
7487 *
7488 * @param pVCpu The cross context virtual CPU structure.
7489 */
7490DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7491{
7492 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7493 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7494 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7495 AssertRC(rc);
7496 Log4(("Cleared NMI-window exiting\n"));
7497}
7498
7499
7500/**
7501 * Evaluates the event to be delivered to the guest and sets it as the pending
7502 * event.
7503 *
7504 * @returns The VT-x guest-interruptibility state.
7505 * @param pVCpu The cross context virtual CPU structure.
7506 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7507 * out-of-sync. Make sure to update the required fields
7508 * before using them.
7509 */
7510static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7511{
7512 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7513 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7514 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7515 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7516 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7517
7518 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7519 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7520 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7521 Assert(!TRPMHasTrap(pVCpu));
7522
7523#ifdef VBOX_WITH_NEW_APIC
7524 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7525 APICUpdatePendingInterrupts(pVCpu);
7526#endif
7527
7528 /*
7529 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7530 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7531 */
7532 /** @todo SMI. SMIs take priority over NMIs. */
7533 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7534 {
7535 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7536 if ( !pVCpu->hm.s.Event.fPending
7537 && !fBlockNmi
7538 && !fBlockSti
7539 && !fBlockMovSS)
7540 {
7541 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7542 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7543 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7544
7545 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7546 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7547 }
7548 else
7549 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7550 }
7551 /*
7552 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7553 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7554 */
7555 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7556 && !pVCpu->hm.s.fSingleInstruction)
7557 {
7558 Assert(!DBGFIsStepping(pVCpu));
7559 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7560 AssertRC(rc);
7561 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7562 if ( !pVCpu->hm.s.Event.fPending
7563 && !fBlockInt
7564 && !fBlockSti
7565 && !fBlockMovSS)
7566 {
7567 uint8_t u8Interrupt;
7568 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7569 if (RT_SUCCESS(rc))
7570 {
7571 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7572 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7573 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7574
7575 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7576 }
7577 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7578 {
7579 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7580 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7581 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7582 }
7583 else
7584 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7585 }
7586 else
7587 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7588 }
7589
7590 return uIntrState;
7591}
7592
7593
7594/**
7595 * Sets a pending-debug exception to be delivered to the guest if the guest is
7596 * single-stepping in the VMCS.
7597 *
7598 * @param pVCpu The cross context virtual CPU structure.
7599 */
7600DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7601{
7602 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7603 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7604 AssertRC(rc);
7605}
7606
7607
7608/**
7609 * Injects any pending events into the guest if the guest is in a state to
7610 * receive them.
7611 *
7612 * @returns Strict VBox status code (i.e. informational status codes too).
7613 * @param pVCpu The cross context virtual CPU structure.
7614 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7615 * out-of-sync. Make sure to update the required fields
7616 * before using them.
7617 * @param uIntrState The VT-x guest-interruptibility state.
7618 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7619 * return VINF_EM_DBG_STEPPED if the event was
7620 * dispatched directly.
7621 */
7622static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7623{
7624 HMVMX_ASSERT_PREEMPT_SAFE();
7625 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7626
7627 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7628 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7629
7630 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7631 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7632 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7633 Assert(!TRPMHasTrap(pVCpu));
7634
7635 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7636 if (pVCpu->hm.s.Event.fPending)
7637 {
7638 /*
7639 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7640 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7641 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7642 *
7643 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7644 */
7645 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7646#ifdef VBOX_STRICT
7647 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7648 {
7649 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7650 Assert(!fBlockInt);
7651 Assert(!fBlockSti);
7652 Assert(!fBlockMovSS);
7653 }
7654 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7655 {
7656 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7657 Assert(!fBlockSti);
7658 Assert(!fBlockMovSS);
7659 Assert(!fBlockNmi);
7660 }
7661#endif
7662 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7663 (uint8_t)uIntType));
7664 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7665 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7666 fStepping, &uIntrState);
7667 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7668
7669 /* Update the interruptibility-state as it could have been changed by
7670 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7671 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7672 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7673
7674 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7675 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7676 else
7677 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7678 }
7679
7680 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7681 if ( fBlockSti
7682 || fBlockMovSS)
7683 {
7684 if (!pVCpu->hm.s.fSingleInstruction)
7685 {
7686 /*
7687 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7688 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7689 * See Intel spec. 27.3.4 "Saving Non-Register State".
7690 */
7691 Assert(!DBGFIsStepping(pVCpu));
7692 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7693 AssertRCReturn(rc2, rc2);
7694 if (pMixedCtx->eflags.Bits.u1TF)
7695 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7696 }
7697 else if (pMixedCtx->eflags.Bits.u1TF)
7698 {
7699 /*
7700 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7701 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7702 */
7703 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7704 uIntrState = 0;
7705 }
7706 }
7707
7708 /*
7709 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7710 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7711 */
7712 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7713 AssertRC(rc2);
7714
7715 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7716 NOREF(fBlockMovSS); NOREF(fBlockSti);
7717 return rcStrict;
7718}
7719
7720
7721/**
7722 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7723 *
7724 * @param pVCpu The cross context virtual CPU structure.
7725 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7726 * out-of-sync. Make sure to update the required fields
7727 * before using them.
7728 */
7729DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7730{
7731 NOREF(pMixedCtx);
7732 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7733 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7734}
7735
7736
7737/**
7738 * Injects a double-fault (\#DF) exception into the VM.
7739 *
7740 * @returns Strict VBox status code (i.e. informational status codes too).
7741 * @param pVCpu The cross context virtual CPU structure.
7742 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7743 * out-of-sync. Make sure to update the required fields
7744 * before using them.
7745 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7746 * and should return VINF_EM_DBG_STEPPED if the event
7747 * is injected directly (register modified by us, not
7748 * by hardware on VM-entry).
7749 * @param puIntrState Pointer to the current guest interruptibility-state.
7750 * This interruptibility-state will be updated if
7751 * necessary. This cannot not be NULL.
7752 */
7753DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7754{
7755 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7756 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7757 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7758 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7759 fStepping, puIntrState);
7760}
7761
7762
7763/**
7764 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7765 *
7766 * @param pVCpu The cross context virtual CPU structure.
7767 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7768 * out-of-sync. Make sure to update the required fields
7769 * before using them.
7770 */
7771DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7772{
7773 NOREF(pMixedCtx);
7774 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7775 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7776 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7777}
7778
7779
7780/**
7781 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7782 *
7783 * @param pVCpu The cross context virtual CPU structure.
7784 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7785 * out-of-sync. Make sure to update the required fields
7786 * before using them.
7787 * @param cbInstr The value of RIP that is to be pushed on the guest
7788 * stack.
7789 */
7790DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7791{
7792 NOREF(pMixedCtx);
7793 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7794 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7795 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7796}
7797
7798
7799/**
7800 * Injects a general-protection (\#GP) fault into the VM.
7801 *
7802 * @returns Strict VBox status code (i.e. informational status codes too).
7803 * @param pVCpu The cross context virtual CPU structure.
7804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7805 * out-of-sync. Make sure to update the required fields
7806 * before using them.
7807 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7808 * mode, i.e. in real-mode it's not valid).
7809 * @param u32ErrorCode The error code associated with the \#GP.
7810 * @param fStepping Whether we're running in
7811 * hmR0VmxRunGuestCodeStep() and should return
7812 * VINF_EM_DBG_STEPPED if the event is injected
7813 * directly (register modified by us, not by
7814 * hardware on VM-entry).
7815 * @param puIntrState Pointer to the current guest interruptibility-state.
7816 * This interruptibility-state will be updated if
7817 * necessary. This cannot not be NULL.
7818 */
7819DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7820 bool fStepping, uint32_t *puIntrState)
7821{
7822 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7823 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7824 if (fErrorCodeValid)
7825 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7826 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7827 fStepping, puIntrState);
7828}
7829
7830
7831/**
7832 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7833 * VM.
7834 *
7835 * @param pVCpu The cross context virtual CPU structure.
7836 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7837 * out-of-sync. Make sure to update the required fields
7838 * before using them.
7839 * @param u32ErrorCode The error code associated with the \#GP.
7840 */
7841DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7842{
7843 NOREF(pMixedCtx);
7844 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7845 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7846 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7847 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7848}
7849
7850
7851/**
7852 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7853 *
7854 * @param pVCpu The cross context virtual CPU structure.
7855 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7856 * out-of-sync. Make sure to update the required fields
7857 * before using them.
7858 * @param uVector The software interrupt vector number.
7859 * @param cbInstr The value of RIP that is to be pushed on the guest
7860 * stack.
7861 */
7862DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7863{
7864 NOREF(pMixedCtx);
7865 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7866 if ( uVector == X86_XCPT_BP
7867 || uVector == X86_XCPT_OF)
7868 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7869 else
7870 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7871 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7872}
7873
7874
7875/**
7876 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7877 * stack.
7878 *
7879 * @returns Strict VBox status code (i.e. informational status codes too).
7880 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7881 * @param pVM The cross context VM structure.
7882 * @param pMixedCtx Pointer to the guest-CPU context.
7883 * @param uValue The value to push to the guest stack.
7884 */
7885DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7886{
7887 /*
7888 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7889 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7890 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7891 */
7892 if (pMixedCtx->sp == 1)
7893 return VINF_EM_RESET;
7894 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7895 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7896 AssertRC(rc);
7897 return rc;
7898}
7899
7900
7901/**
7902 * Injects an event into the guest upon VM-entry by updating the relevant fields
7903 * in the VM-entry area in the VMCS.
7904 *
7905 * @returns Strict VBox status code (i.e. informational status codes too).
7906 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7907 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7908 *
7909 * @param pVCpu The cross context virtual CPU structure.
7910 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7911 * be out-of-sync. Make sure to update the required
7912 * fields before using them.
7913 * @param u64IntInfo The VM-entry interruption-information field.
7914 * @param cbInstr The VM-entry instruction length in bytes (for
7915 * software interrupts, exceptions and privileged
7916 * software exceptions).
7917 * @param u32ErrCode The VM-entry exception error code.
7918 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7919 * @param puIntrState Pointer to the current guest interruptibility-state.
7920 * This interruptibility-state will be updated if
7921 * necessary. This cannot not be NULL.
7922 * @param fStepping Whether we're running in
7923 * hmR0VmxRunGuestCodeStep() and should return
7924 * VINF_EM_DBG_STEPPED if the event is injected
7925 * directly (register modified by us, not by
7926 * hardware on VM-entry).
7927 *
7928 * @remarks Requires CR0!
7929 * @remarks No-long-jump zone!!!
7930 */
7931static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7932 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7933 uint32_t *puIntrState)
7934{
7935 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7936 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7937 Assert(puIntrState);
7938 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7939
7940 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7941 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7942
7943#ifdef VBOX_STRICT
7944 /* Validate the error-code-valid bit for hardware exceptions. */
7945 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7946 {
7947 switch (uVector)
7948 {
7949 case X86_XCPT_PF:
7950 case X86_XCPT_DF:
7951 case X86_XCPT_TS:
7952 case X86_XCPT_NP:
7953 case X86_XCPT_SS:
7954 case X86_XCPT_GP:
7955 case X86_XCPT_AC:
7956 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7957 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7958 /* fallthru */
7959 default:
7960 break;
7961 }
7962 }
7963#endif
7964
7965 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7966 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7967 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7968
7969 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7970
7971 /* We require CR0 to check if the guest is in real-mode. */
7972 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7973 AssertRCReturn(rc, rc);
7974
7975 /*
7976 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7977 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7978 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7979 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7980 */
7981 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7982 {
7983 PVM pVM = pVCpu->CTX_SUFF(pVM);
7984 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7985 {
7986 Assert(PDMVmmDevHeapIsEnabled(pVM));
7987 Assert(pVM->hm.s.vmx.pRealModeTSS);
7988
7989 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7990 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7991 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7992 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7993 AssertRCReturn(rc, rc);
7994 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7995
7996 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7997 size_t const cbIdtEntry = sizeof(X86IDTR16);
7998 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7999 {
8000 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8001 if (uVector == X86_XCPT_DF)
8002 return VINF_EM_RESET;
8003
8004 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8005 if (uVector == X86_XCPT_GP)
8006 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8007
8008 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8009 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8010 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8011 fStepping, puIntrState);
8012 }
8013
8014 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8015 uint16_t uGuestIp = pMixedCtx->ip;
8016 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8017 {
8018 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8019 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8020 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8021 }
8022 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8023 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8024
8025 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8026 X86IDTR16 IdtEntry;
8027 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8028 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8029 AssertRCReturn(rc, rc);
8030
8031 /* Construct the stack frame for the interrupt/exception handler. */
8032 VBOXSTRICTRC rcStrict;
8033 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8034 if (rcStrict == VINF_SUCCESS)
8035 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8036 if (rcStrict == VINF_SUCCESS)
8037 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8038
8039 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8040 if (rcStrict == VINF_SUCCESS)
8041 {
8042 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8043 pMixedCtx->rip = IdtEntry.offSel;
8044 pMixedCtx->cs.Sel = IdtEntry.uSel;
8045 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8046 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8047 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8048 && uVector == X86_XCPT_PF)
8049 pMixedCtx->cr2 = GCPtrFaultAddress;
8050
8051 /* If any other guest-state bits are changed here, make sure to update
8052 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8053 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8054 | HM_CHANGED_GUEST_RIP
8055 | HM_CHANGED_GUEST_RFLAGS
8056 | HM_CHANGED_GUEST_RSP);
8057
8058 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8059 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8060 {
8061 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8062 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8063 Log4(("Clearing inhibition due to STI.\n"));
8064 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8065 }
8066 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8067 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8068
8069 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8070 it, if we are returning to ring-3 before executing guest code. */
8071 pVCpu->hm.s.Event.fPending = false;
8072
8073 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8074 if (fStepping)
8075 rcStrict = VINF_EM_DBG_STEPPED;
8076 }
8077 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8078 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8079 return rcStrict;
8080 }
8081
8082 /*
8083 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8084 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8085 */
8086 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8087 }
8088
8089 /* Validate. */
8090 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8091 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8092 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8093
8094 /* Inject. */
8095 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8096 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8097 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8098 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8099
8100 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8101 && uVector == X86_XCPT_PF)
8102 pMixedCtx->cr2 = GCPtrFaultAddress;
8103
8104 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8105 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8106
8107 AssertRCReturn(rc, rc);
8108 return VINF_SUCCESS;
8109}
8110
8111
8112/**
8113 * Clears the interrupt-window exiting control in the VMCS and if necessary
8114 * clears the current event in the VMCS as well.
8115 *
8116 * @returns VBox status code.
8117 * @param pVCpu The cross context virtual CPU structure.
8118 *
8119 * @remarks Use this function only to clear events that have not yet been
8120 * delivered to the guest but are injected in the VMCS!
8121 * @remarks No-long-jump zone!!!
8122 */
8123static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8124{
8125 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8126
8127 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8128 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8129
8130 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8131 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8132}
8133
8134
8135/**
8136 * Enters the VT-x session.
8137 *
8138 * @returns VBox status code.
8139 * @param pVM The cross context VM structure.
8140 * @param pVCpu The cross context virtual CPU structure.
8141 * @param pCpu Pointer to the CPU info struct.
8142 */
8143VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8144{
8145 AssertPtr(pVM);
8146 AssertPtr(pVCpu);
8147 Assert(pVM->hm.s.vmx.fSupported);
8148 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8149 NOREF(pCpu); NOREF(pVM);
8150
8151 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8152 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8153
8154#ifdef VBOX_STRICT
8155 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8156 RTCCUINTREG uHostCR4 = ASMGetCR4();
8157 if (!(uHostCR4 & X86_CR4_VMXE))
8158 {
8159 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8160 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8161 }
8162#endif
8163
8164 /*
8165 * Load the VCPU's VMCS as the current (and active) one.
8166 */
8167 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8168 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8169 if (RT_FAILURE(rc))
8170 return rc;
8171
8172 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8173 pVCpu->hm.s.fLeaveDone = false;
8174 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8175
8176 return VINF_SUCCESS;
8177}
8178
8179
8180/**
8181 * The thread-context callback (only on platforms which support it).
8182 *
8183 * @param enmEvent The thread-context event.
8184 * @param pVCpu The cross context virtual CPU structure.
8185 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8186 * @thread EMT(pVCpu)
8187 */
8188VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8189{
8190 NOREF(fGlobalInit);
8191
8192 switch (enmEvent)
8193 {
8194 case RTTHREADCTXEVENT_OUT:
8195 {
8196 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8197 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8198 VMCPU_ASSERT_EMT(pVCpu);
8199
8200 PVM pVM = pVCpu->CTX_SUFF(pVM);
8201 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8202
8203 /* No longjmps (logger flushes, locks) in this fragile context. */
8204 VMMRZCallRing3Disable(pVCpu);
8205 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8206
8207 /*
8208 * Restore host-state (FPU, debug etc.)
8209 */
8210 if (!pVCpu->hm.s.fLeaveDone)
8211 {
8212 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8213 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8214 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8215 pVCpu->hm.s.fLeaveDone = true;
8216 }
8217
8218 /* Leave HM context, takes care of local init (term). */
8219 int rc = HMR0LeaveCpu(pVCpu);
8220 AssertRC(rc); NOREF(rc);
8221
8222 /* Restore longjmp state. */
8223 VMMRZCallRing3Enable(pVCpu);
8224 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8225 break;
8226 }
8227
8228 case RTTHREADCTXEVENT_IN:
8229 {
8230 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8231 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8232 VMCPU_ASSERT_EMT(pVCpu);
8233
8234 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8235 VMMRZCallRing3Disable(pVCpu);
8236 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8237
8238 /* Initialize the bare minimum state required for HM. This takes care of
8239 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8240 int rc = HMR0EnterCpu(pVCpu);
8241 AssertRC(rc);
8242 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8243
8244 /* Load the active VMCS as the current one. */
8245 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8246 {
8247 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8248 AssertRC(rc); NOREF(rc);
8249 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8250 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8251 }
8252 pVCpu->hm.s.fLeaveDone = false;
8253
8254 /* Restore longjmp state. */
8255 VMMRZCallRing3Enable(pVCpu);
8256 break;
8257 }
8258
8259 default:
8260 break;
8261 }
8262}
8263
8264
8265/**
8266 * Saves the host state in the VMCS host-state.
8267 * Sets up the VM-exit MSR-load area.
8268 *
8269 * The CPU state will be loaded from these fields on every successful VM-exit.
8270 *
8271 * @returns VBox status code.
8272 * @param pVM The cross context VM structure.
8273 * @param pVCpu The cross context virtual CPU structure.
8274 *
8275 * @remarks No-long-jump zone!!!
8276 */
8277static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8278{
8279 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8280
8281 int rc = VINF_SUCCESS;
8282 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8283 {
8284 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8285 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8286
8287 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8288 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8289
8290 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8291 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8292
8293 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8294 }
8295 return rc;
8296}
8297
8298
8299/**
8300 * Saves the host state in the VMCS host-state.
8301 *
8302 * @returns VBox status code.
8303 * @param pVM The cross context VM structure.
8304 * @param pVCpu The cross context virtual CPU structure.
8305 *
8306 * @remarks No-long-jump zone!!!
8307 */
8308VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8309{
8310 AssertPtr(pVM);
8311 AssertPtr(pVCpu);
8312
8313 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8314
8315 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8316 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8317 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8318 return hmR0VmxSaveHostState(pVM, pVCpu);
8319}
8320
8321
8322/**
8323 * Loads the guest state into the VMCS guest-state area.
8324 *
8325 * The will typically be done before VM-entry when the guest-CPU state and the
8326 * VMCS state may potentially be out of sync.
8327 *
8328 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8329 * VM-entry controls.
8330 * Sets up the appropriate VMX non-root function to execute guest code based on
8331 * the guest CPU mode.
8332 *
8333 * @returns VBox strict status code.
8334 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8335 * without unrestricted guest access and the VMMDev is not presently
8336 * mapped (e.g. EFI32).
8337 *
8338 * @param pVM The cross context VM structure.
8339 * @param pVCpu The cross context virtual CPU structure.
8340 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8341 * out-of-sync. Make sure to update the required fields
8342 * before using them.
8343 *
8344 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8345 * caller disables then again on successfull return. Confusing.)
8346 */
8347static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8348{
8349 AssertPtr(pVM);
8350 AssertPtr(pVCpu);
8351 AssertPtr(pMixedCtx);
8352 HMVMX_ASSERT_PREEMPT_SAFE();
8353
8354 VMMRZCallRing3Disable(pVCpu);
8355 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8356
8357 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8358
8359 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8360
8361 /* Determine real-on-v86 mode. */
8362 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8363 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8364 && CPUMIsGuestInRealModeEx(pMixedCtx))
8365 {
8366 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8367 }
8368
8369 /*
8370 * Load the guest-state into the VMCS.
8371 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8372 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8373 */
8374 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8375 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8376
8377 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8378 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8379 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8380
8381 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8382 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8383 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8384
8385 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8386 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8387
8388 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8389 if (rcStrict == VINF_SUCCESS)
8390 { /* likely */ }
8391 else
8392 {
8393 VMMRZCallRing3Enable(pVCpu);
8394 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8395 return rcStrict;
8396 }
8397
8398 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8399 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8400 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8401
8402 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8403 determine we don't have to swap EFER after all. */
8404 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8405 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8406
8407 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8408 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8409
8410 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8411 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8412
8413 /*
8414 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8415 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8416 */
8417 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8418 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8419
8420 /* Clear any unused and reserved bits. */
8421 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8422
8423 VMMRZCallRing3Enable(pVCpu);
8424
8425 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8426 return rc;
8427}
8428
8429
8430/**
8431 * Loads the state shared between the host and guest into the VMCS.
8432 *
8433 * @param pVM The cross context VM structure.
8434 * @param pVCpu The cross context virtual CPU structure.
8435 * @param pCtx Pointer to the guest-CPU context.
8436 *
8437 * @remarks No-long-jump zone!!!
8438 */
8439static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8440{
8441 NOREF(pVM);
8442
8443 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8444 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8445
8446 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8447 {
8448 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8449 AssertRC(rc);
8450 }
8451
8452 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8453 {
8454 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8455 AssertRC(rc);
8456
8457 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8458 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8459 {
8460 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8461 AssertRC(rc);
8462 }
8463 }
8464
8465 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8466 {
8467 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8468 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8469 }
8470
8471 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8472 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8473 {
8474 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8475 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8476 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8477 AssertRC(rc);
8478 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8479 }
8480
8481 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8482 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8483}
8484
8485
8486/**
8487 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8488 *
8489 * @returns Strict VBox status code (i.e. informational status codes too).
8490 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8491 * without unrestricted guest access and the VMMDev is not presently
8492 * mapped (e.g. EFI32).
8493 *
8494 * @param pVM The cross context VM structure.
8495 * @param pVCpu The cross context virtual CPU structure.
8496 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8497 * out-of-sync. Make sure to update the required fields
8498 * before using them.
8499 */
8500static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8501{
8502 HMVMX_ASSERT_PREEMPT_SAFE();
8503
8504 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8505#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8506 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8507#endif
8508
8509 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8510 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8511 {
8512 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8513 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8514 { /* likely */}
8515 else
8516 {
8517 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8518 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8519 }
8520 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8521 }
8522 else if (HMCPU_CF_VALUE(pVCpu))
8523 {
8524 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8525 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8526 { /* likely */}
8527 else
8528 {
8529 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8530 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8531 return rcStrict;
8532 }
8533 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8534 }
8535
8536 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8537 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8538 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8539 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8540 return rcStrict;
8541}
8542
8543
8544/**
8545 * Does the preparations before executing guest code in VT-x.
8546 *
8547 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8548 * recompiler/IEM. We must be cautious what we do here regarding committing
8549 * guest-state information into the VMCS assuming we assuredly execute the
8550 * guest in VT-x mode.
8551 *
8552 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8553 * the common-state (TRPM/forceflags), we must undo those changes so that the
8554 * recompiler/IEM can (and should) use them when it resumes guest execution.
8555 * Otherwise such operations must be done when we can no longer exit to ring-3.
8556 *
8557 * @returns Strict VBox status code (i.e. informational status codes too).
8558 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8559 * have been disabled.
8560 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8561 * double-fault into the guest.
8562 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8563 * dispatched directly.
8564 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8565 *
8566 * @param pVM The cross context VM structure.
8567 * @param pVCpu The cross context virtual CPU structure.
8568 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8569 * out-of-sync. Make sure to update the required fields
8570 * before using them.
8571 * @param pVmxTransient Pointer to the VMX transient structure.
8572 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8573 * us ignore some of the reasons for returning to
8574 * ring-3, and return VINF_EM_DBG_STEPPED if event
8575 * dispatching took place.
8576 */
8577static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8578{
8579 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8580
8581#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8582 PGMRZDynMapFlushAutoSet(pVCpu);
8583#endif
8584
8585 /* Check force flag actions that might require us to go back to ring-3. */
8586 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8587 if (rcStrict == VINF_SUCCESS)
8588 { /* FFs doesn't get set all the time. */ }
8589 else
8590 return rcStrict;
8591
8592#ifndef IEM_VERIFICATION_MODE_FULL
8593 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8594 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8595 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8596 {
8597 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8598 RTGCPHYS GCPhysApicBase;
8599 GCPhysApicBase = pMixedCtx->msrApicBase;
8600 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8601
8602 /* Unalias any existing mapping. */
8603 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8604 AssertRCReturn(rc, rc);
8605
8606 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8607 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8608 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8609 AssertRCReturn(rc, rc);
8610
8611 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8612 }
8613#endif /* !IEM_VERIFICATION_MODE_FULL */
8614
8615 if (TRPMHasTrap(pVCpu))
8616 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8617 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8618
8619 /*
8620 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8621 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8622 */
8623 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8624 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8625 { /* likely */ }
8626 else
8627 {
8628 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8629 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8630 return rcStrict;
8631 }
8632
8633 /*
8634 * Load the guest state bits, we can handle longjmps/getting preempted here.
8635 *
8636 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8637 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8638 * Hence, this needs to be done -after- injection of events.
8639 */
8640 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8641 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8642 { /* likely */ }
8643 else
8644 return rcStrict;
8645
8646 /*
8647 * No longjmps to ring-3 from this point on!!!
8648 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8649 * This also disables flushing of the R0-logger instance (if any).
8650 */
8651 VMMRZCallRing3Disable(pVCpu);
8652
8653 /*
8654 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8655 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8656 *
8657 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8658 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8659 *
8660 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8661 * executing guest code.
8662 */
8663 pVmxTransient->fEFlags = ASMIntDisableFlags();
8664
8665 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8666 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8667 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8668 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8669 {
8670 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8671 {
8672 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8673 pVCpu->hm.s.Event.fPending = false;
8674
8675 return VINF_SUCCESS;
8676 }
8677
8678 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8679 rcStrict = VINF_EM_RAW_INTERRUPT;
8680 }
8681 else
8682 {
8683 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8684 rcStrict = VINF_EM_RAW_TO_R3;
8685 }
8686
8687 ASMSetFlags(pVmxTransient->fEFlags);
8688 VMMRZCallRing3Enable(pVCpu);
8689
8690 return rcStrict;
8691}
8692
8693
8694/**
8695 * Prepares to run guest code in VT-x and we've committed to doing so. This
8696 * means there is no backing out to ring-3 or anywhere else at this
8697 * point.
8698 *
8699 * @param pVM The cross context VM structure.
8700 * @param pVCpu The cross context virtual CPU structure.
8701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8702 * out-of-sync. Make sure to update the required fields
8703 * before using them.
8704 * @param pVmxTransient Pointer to the VMX transient structure.
8705 *
8706 * @remarks Called with preemption disabled.
8707 * @remarks No-long-jump zone!!!
8708 */
8709static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8710{
8711 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8712 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8713 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8714
8715 /*
8716 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8717 */
8718 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8719 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8720
8721#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8722 if (!CPUMIsGuestFPUStateActive(pVCpu))
8723 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8724 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8725 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8726#endif
8727
8728 if ( pVCpu->hm.s.fPreloadGuestFpu
8729 && !CPUMIsGuestFPUStateActive(pVCpu))
8730 {
8731 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8732 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8733 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8734 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8735 }
8736
8737 /*
8738 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8739 */
8740 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8741 && pVCpu->hm.s.vmx.cMsrs > 0)
8742 {
8743 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8744 }
8745
8746 /*
8747 * Load the host state bits as we may've been preempted (only happens when
8748 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8749 */
8750 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8751 * any effect to the host state needing to be saved? */
8752 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8753 {
8754 /* This ASSUMES that pfnStartVM has been set up already. */
8755 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8756 AssertRC(rc);
8757 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8758 }
8759 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8760
8761 /*
8762 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8763 */
8764 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8765 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8766 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8767
8768 /* Store status of the shared guest-host state at the time of VM-entry. */
8769#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8770 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8771 {
8772 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8773 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8774 }
8775 else
8776#endif
8777 {
8778 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8779 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8780 }
8781 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8782
8783 /*
8784 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8785 */
8786 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8787 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8788
8789 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8790 RTCPUID idCurrentCpu = pCpu->idCpu;
8791 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8792 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8793 {
8794 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8795 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8796 }
8797
8798 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8799 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8800 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8801 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8802
8803 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8804
8805 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8806 to start executing. */
8807
8808 /*
8809 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8810 */
8811 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8812 {
8813 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8814 {
8815 bool fMsrUpdated;
8816 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8817 AssertRC(rc2);
8818 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8819
8820 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8821 &fMsrUpdated);
8822 AssertRC(rc2);
8823 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8824
8825 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8826 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8827 }
8828 else
8829 {
8830 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8831 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8832 }
8833 }
8834
8835#ifdef VBOX_STRICT
8836 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8837 hmR0VmxCheckHostEferMsr(pVCpu);
8838 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8839#endif
8840#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8841 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8842 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8843 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8844#endif
8845}
8846
8847
8848/**
8849 * Performs some essential restoration of state after running guest code in
8850 * VT-x.
8851 *
8852 * @param pVM The cross context VM structure.
8853 * @param pVCpu The cross context virtual CPU structure.
8854 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8855 * out-of-sync. Make sure to update the required fields
8856 * before using them.
8857 * @param pVmxTransient Pointer to the VMX transient structure.
8858 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8859 *
8860 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8861 *
8862 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8863 * unconditionally when it is safe to do so.
8864 */
8865static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8866{
8867 NOREF(pVM);
8868
8869 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8870
8871 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8872 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8873 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8874 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8875 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8876 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8877
8878 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8879 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8880
8881 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8882 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8883 Assert(!ASMIntAreEnabled());
8884 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8885
8886#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8887 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8888 {
8889 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8890 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8891 }
8892#endif
8893
8894#if HC_ARCH_BITS == 64
8895 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8896#endif
8897 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8898#ifdef VBOX_STRICT
8899 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8900#endif
8901 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8902 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8903
8904 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8905 uint32_t uExitReason;
8906 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8907 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8908 AssertRC(rc);
8909 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8910 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8911
8912 /* Update the VM-exit history array. */
8913 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8914
8915 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8916 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8917 {
8918 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8919 pVmxTransient->fVMEntryFailed));
8920 return;
8921 }
8922
8923 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8924 {
8925 /** @todo We can optimize this by only syncing with our force-flags when
8926 * really needed and keeping the VMCS state as it is for most
8927 * VM-exits. */
8928 /* Update the guest interruptibility-state from the VMCS. */
8929 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8930
8931#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8932 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8933 AssertRC(rc);
8934#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8935 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8936 AssertRC(rc);
8937#endif
8938
8939 /*
8940 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8941 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8942 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8943 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8944 */
8945 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8946 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8947 {
8948 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8949 AssertRC(rc);
8950 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8951 }
8952 }
8953}
8954
8955
8956/**
8957 * Runs the guest code using VT-x the normal way.
8958 *
8959 * @returns VBox status code.
8960 * @param pVM The cross context VM structure.
8961 * @param pVCpu The cross context virtual CPU structure.
8962 * @param pCtx Pointer to the guest-CPU context.
8963 *
8964 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8965 */
8966static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8967{
8968 VMXTRANSIENT VmxTransient;
8969 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8970 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8971 uint32_t cLoops = 0;
8972
8973 for (;; cLoops++)
8974 {
8975 Assert(!HMR0SuspendPending());
8976 HMVMX_ASSERT_CPU_SAFE();
8977
8978 /* Preparatory work for running guest code, this may force us to return
8979 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8980 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8981 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8982 if (rcStrict != VINF_SUCCESS)
8983 break;
8984
8985 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8986 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8987 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8988
8989 /* Restore any residual host-state and save any bits shared between host
8990 and guest into the guest-CPU state. Re-enables interrupts! */
8991 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8992
8993 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8994 if (RT_SUCCESS(rcRun))
8995 { /* very likely */ }
8996 else
8997 {
8998 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8999 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9000 return rcRun;
9001 }
9002
9003 /* Profile the VM-exit. */
9004 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9005 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9006 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9007 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9008 HMVMX_START_EXIT_DISPATCH_PROF();
9009
9010 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9011
9012 /* Handle the VM-exit. */
9013#ifdef HMVMX_USE_FUNCTION_TABLE
9014 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9015#else
9016 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9017#endif
9018 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9019 if (rcStrict == VINF_SUCCESS)
9020 {
9021 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9022 continue; /* likely */
9023 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9024 rcStrict = VINF_EM_RAW_INTERRUPT;
9025 }
9026 break;
9027 }
9028
9029 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9030 return rcStrict;
9031}
9032
9033
9034
9035/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9036 * probes.
9037 *
9038 * The following few functions and associated structure contains the bloat
9039 * necessary for providing detailed debug events and dtrace probes as well as
9040 * reliable host side single stepping. This works on the principle of
9041 * "subclassing" the normal execution loop and workers. We replace the loop
9042 * method completely and override selected helpers to add necessary adjustments
9043 * to their core operation.
9044 *
9045 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9046 * any performance for debug and analysis features.
9047 *
9048 * @{
9049 */
9050
9051typedef struct VMXRUNDBGSTATE
9052{
9053 /** The RIP we started executing at. This is for detecting that we stepped. */
9054 uint64_t uRipStart;
9055 /** The CS we started executing with. */
9056 uint16_t uCsStart;
9057
9058 /** Whether we've actually modified the 1st execution control field. */
9059 bool fModifiedProcCtls : 1;
9060 /** Whether we've actually modified the 2nd execution control field. */
9061 bool fModifiedProcCtls2 : 1;
9062 /** Whether we've actually modified the exception bitmap. */
9063 bool fModifiedXcptBitmap : 1;
9064
9065 /** We desire the modified the CR0 mask to be cleared. */
9066 bool fClearCr0Mask : 1;
9067 /** We desire the modified the CR4 mask to be cleared. */
9068 bool fClearCr4Mask : 1;
9069 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9070 uint32_t fCpe1Extra;
9071 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9072 uint32_t fCpe1Unwanted;
9073 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9074 uint32_t fCpe2Extra;
9075 /** Extra stuff we need in */
9076 uint32_t bmXcptExtra;
9077 /** The sequence number of the Dtrace provider settings the state was
9078 * configured against. */
9079 uint32_t uDtraceSettingsSeqNo;
9080 /** Exits to check (one bit per exit). */
9081 uint32_t bmExitsToCheck[3];
9082
9083 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9084 uint32_t fProcCtlsInitial;
9085 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9086 uint32_t fProcCtls2Initial;
9087 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9088 uint32_t bmXcptInitial;
9089} VMXRUNDBGSTATE;
9090AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9091typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9092
9093
9094/**
9095 * Initializes the VMXRUNDBGSTATE structure.
9096 *
9097 * @param pVCpu The cross context virtual CPU structure of the
9098 * calling EMT.
9099 * @param pCtx The CPU register context to go with @a pVCpu.
9100 * @param pDbgState The structure to initialize.
9101 */
9102DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9103{
9104 pDbgState->uRipStart = pCtx->rip;
9105 pDbgState->uCsStart = pCtx->cs.Sel;
9106
9107 pDbgState->fModifiedProcCtls = false;
9108 pDbgState->fModifiedProcCtls2 = false;
9109 pDbgState->fModifiedXcptBitmap = false;
9110 pDbgState->fClearCr0Mask = false;
9111 pDbgState->fClearCr4Mask = false;
9112 pDbgState->fCpe1Extra = 0;
9113 pDbgState->fCpe1Unwanted = 0;
9114 pDbgState->fCpe2Extra = 0;
9115 pDbgState->bmXcptExtra = 0;
9116 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9117 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9118 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9119}
9120
9121
9122/**
9123 * Updates the VMSC fields with changes requested by @a pDbgState.
9124 *
9125 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9126 * immediately before executing guest code, i.e. when interrupts are disabled.
9127 * We don't check status codes here as we cannot easily assert or return in the
9128 * latter case.
9129 *
9130 * @param pVCpu The cross context virtual CPU structure.
9131 * @param pDbgState The debug state.
9132 */
9133DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9134{
9135 /*
9136 * Ensure desired flags in VMCS control fields are set.
9137 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9138 *
9139 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9140 * there should be no stale data in pCtx at this point.
9141 */
9142 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9143 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9144 {
9145 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9146 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9147 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9148 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9149 pDbgState->fModifiedProcCtls = true;
9150 }
9151
9152 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9153 {
9154 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9155 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9156 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9157 pDbgState->fModifiedProcCtls2 = true;
9158 }
9159
9160 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9161 {
9162 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9163 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9164 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9165 pDbgState->fModifiedXcptBitmap = true;
9166 }
9167
9168 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9169 {
9170 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9171 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9172 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9173 }
9174
9175 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9176 {
9177 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9178 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9179 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9180 }
9181}
9182
9183
9184DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9185{
9186 /*
9187 * Restore exit control settings as we may not reenter this function the
9188 * next time around.
9189 */
9190 /* We reload the initial value, trigger what we can of recalculations the
9191 next time around. From the looks of things, that's all that's required atm. */
9192 if (pDbgState->fModifiedProcCtls)
9193 {
9194 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9195 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9196 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9197 AssertRCReturn(rc2, rc2);
9198 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9199 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9200 }
9201
9202 /* We're currently the only ones messing with this one, so just restore the
9203 cached value and reload the field. */
9204 if ( pDbgState->fModifiedProcCtls2
9205 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9206 {
9207 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9208 AssertRCReturn(rc2, rc2);
9209 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9210 }
9211
9212 /* If we've modified the exception bitmap, we restore it and trigger
9213 reloading and partial recalculation the next time around. */
9214 if (pDbgState->fModifiedXcptBitmap)
9215 {
9216 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9217 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9218 }
9219
9220 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9221 if (pDbgState->fClearCr0Mask)
9222 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9223
9224 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9225 if (pDbgState->fClearCr4Mask)
9226 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9227
9228 return rcStrict;
9229}
9230
9231
9232/**
9233 * Configures VM-exit controls for current DBGF and DTrace settings.
9234 *
9235 * This updates @a pDbgState and the VMCS execution control fields to reflect
9236 * the necessary exits demanded by DBGF and DTrace.
9237 *
9238 * @param pVM The cross context VM structure.
9239 * @param pVCpu The cross context virtual CPU structure.
9240 * @param pCtx Pointer to the guest-CPU context.
9241 * @param pDbgState The debug state.
9242 * @param pVmxTransient Pointer to the VMX transient structure. May update
9243 * fUpdateTscOffsettingAndPreemptTimer.
9244 */
9245static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9246 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9247{
9248 /*
9249 * Take down the dtrace serial number so we can spot changes.
9250 */
9251 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9252 ASMCompilerBarrier();
9253
9254 /*
9255 * We'll rebuild most of the middle block of data members (holding the
9256 * current settings) as we go along here, so start by clearing it all.
9257 */
9258 pDbgState->bmXcptExtra = 0;
9259 pDbgState->fCpe1Extra = 0;
9260 pDbgState->fCpe1Unwanted = 0;
9261 pDbgState->fCpe2Extra = 0;
9262 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9263 pDbgState->bmExitsToCheck[i] = 0;
9264
9265 /*
9266 * Software interrupts (INT XXh) - no idea how to trigger these...
9267 */
9268 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9269 || VBOXVMM_INT_SOFTWARE_ENABLED())
9270 {
9271 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9272 }
9273
9274 /*
9275 * Exception bitmap and XCPT events+probes.
9276 */
9277 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9278 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9279 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9280
9281 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9282 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9283 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9284 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9285 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9286 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9287 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9288 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9289 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9290 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9291 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9292 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9293 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9294 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9295 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9296 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9297 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9298 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9299
9300 if (pDbgState->bmXcptExtra)
9301 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9302
9303 /*
9304 * Process events and probes for VM exits, making sure we get the wanted exits.
9305 *
9306 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9307 * So, when adding/changing/removing please don't forget to update it.
9308 *
9309 * Some of the macros are picking up local variables to save horizontal space,
9310 * (being able to see it in a table is the lesser evil here).
9311 */
9312#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9313 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9314 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9315#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9316 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9317 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9318 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9319 } else do { } while (0)
9320#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9321 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9322 { \
9323 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9324 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9325 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9326 } else do { } while (0)
9327#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9328 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9329 { \
9330 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9331 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9332 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9333 } else do { } while (0)
9334#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9335 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9336 { \
9337 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9338 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9339 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9340 } else do { } while (0)
9341
9342 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9343 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9344 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9345 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9346 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9347
9348 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9349 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9350 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9351 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9352 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9353 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9354 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9355 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9356 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9357 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9358 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9359 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9360 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9361 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9362 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9363 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9364 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9365 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9366 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9367 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9368 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9369 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9370 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9371 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9372 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9373 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9374 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9375 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9376 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9377 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9378 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9379 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9380 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9381 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9382 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9383 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9384
9385 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9386 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9387 {
9388 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9389 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9390 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9391 AssertRC(rc2);
9392
9393#if 0 /** @todo fix me */
9394 pDbgState->fClearCr0Mask = true;
9395 pDbgState->fClearCr4Mask = true;
9396#endif
9397 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9398 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9399 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9400 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9401 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9402 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9403 require clearing here and in the loop if we start using it. */
9404 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9405 }
9406 else
9407 {
9408 if (pDbgState->fClearCr0Mask)
9409 {
9410 pDbgState->fClearCr0Mask = false;
9411 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9412 }
9413 if (pDbgState->fClearCr4Mask)
9414 {
9415 pDbgState->fClearCr4Mask = false;
9416 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9417 }
9418 }
9419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9421
9422 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9423 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9424 {
9425 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9426 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9427 }
9428 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9429 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9430
9431 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9432 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9433 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9435 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9436 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9437 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9438 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9439#if 0 /** @todo too slow, fix handler. */
9440 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9441#endif
9442 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9443
9444 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9445 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9446 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9447 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9448 {
9449 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9450 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9451 }
9452 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9453 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9454 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9455 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9456
9457 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9458 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9459 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9460 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9461 {
9462 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9463 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9464 }
9465 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9466 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9467 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9468 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9469
9470 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9471 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9472 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9473 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9474 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9475 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9476 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9477 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9478 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9479 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9480 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9481 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9482 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9483 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9484 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9485 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9486 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9487 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9488 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9489 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9490 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9491 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9492
9493#undef IS_EITHER_ENABLED
9494#undef SET_ONLY_XBM_IF_EITHER_EN
9495#undef SET_CPE1_XBM_IF_EITHER_EN
9496#undef SET_CPEU_XBM_IF_EITHER_EN
9497#undef SET_CPE2_XBM_IF_EITHER_EN
9498
9499 /*
9500 * Sanitize the control stuff.
9501 */
9502 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9503 if (pDbgState->fCpe2Extra)
9504 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9505 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9506 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9507 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9508 {
9509 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9510 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9511 }
9512
9513 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9514 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9515 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9516 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9517}
9518
9519
9520/**
9521 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9522 *
9523 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9524 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9525 * either.
9526 *
9527 * @returns Strict VBox status code (i.e. informational status codes too).
9528 * @param pVM The cross context VM structure.
9529 * @param pVCpu The cross context virtual CPU structure.
9530 * @param pMixedCtx Pointer to the guest-CPU context.
9531 * @param pVmxTransient Pointer to the VMX-transient structure.
9532 * @param uExitReason The VM-exit reason.
9533 *
9534 * @remarks The name of this function is displayed by dtrace, so keep it short
9535 * and to the point. No longer than 33 chars long, please.
9536 */
9537static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9538 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9539{
9540 /*
9541 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9542 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9543 *
9544 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9545 * does. Must add/change/remove both places. Same ordering, please.
9546 *
9547 * Added/removed events must also be reflected in the next section
9548 * where we dispatch dtrace events.
9549 */
9550 bool fDtrace1 = false;
9551 bool fDtrace2 = false;
9552 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9553 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9554 uint32_t uEventArg = 0;
9555#define SET_EXIT(a_EventSubName) \
9556 do { \
9557 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9558 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9559 } while (0)
9560#define SET_BOTH(a_EventSubName) \
9561 do { \
9562 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9563 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9564 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9565 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9566 } while (0)
9567 switch (uExitReason)
9568 {
9569 case VMX_EXIT_MTF:
9570 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9571
9572 case VMX_EXIT_XCPT_OR_NMI:
9573 {
9574 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9575 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9576 {
9577 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9578 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9579 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9580 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9581 {
9582 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9583 {
9584 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9585 uEventArg = pVmxTransient->uExitIntErrorCode;
9586 }
9587 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9588 switch (enmEvent1)
9589 {
9590 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9591 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9592 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9593 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9594 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9595 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9596 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9597 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9598 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9599 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9600 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9601 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9602 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9603 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9604 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9605 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9606 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9607 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9608 default: break;
9609 }
9610 }
9611 else
9612 AssertFailed();
9613 break;
9614
9615 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9616 uEventArg = idxVector;
9617 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9618 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9619 break;
9620 }
9621 break;
9622 }
9623
9624 case VMX_EXIT_TRIPLE_FAULT:
9625 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9626 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9627 break;
9628 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9629 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9630 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9631 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9632 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9633
9634 /* Instruction specific VM-exits: */
9635 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9636 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9637 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9638 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9639 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9640 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9641 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9642 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9643 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9644 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9645 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9646 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9647 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9648 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9649 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9650 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9651 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9652 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9653 case VMX_EXIT_MOV_CRX:
9654 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9655/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9656* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9657 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9658 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9659 SET_BOTH(CRX_READ);
9660 else
9661 SET_BOTH(CRX_WRITE);
9662 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9663 break;
9664 case VMX_EXIT_MOV_DRX:
9665 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9666 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9667 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9668 SET_BOTH(DRX_READ);
9669 else
9670 SET_BOTH(DRX_WRITE);
9671 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9672 break;
9673 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9674 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9675 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9676 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9677 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9678 case VMX_EXIT_XDTR_ACCESS:
9679 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9680 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9681 {
9682 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9683 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9684 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9685 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9686 }
9687 break;
9688
9689 case VMX_EXIT_TR_ACCESS:
9690 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9691 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9692 {
9693 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9694 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9695 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9696 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9697 }
9698 break;
9699
9700 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9701 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9702 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9703 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9704 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9705 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9706 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9707 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9708 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9709 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9710 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9711
9712 /* Events that aren't relevant at this point. */
9713 case VMX_EXIT_EXT_INT:
9714 case VMX_EXIT_INT_WINDOW:
9715 case VMX_EXIT_NMI_WINDOW:
9716 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9717 case VMX_EXIT_PREEMPT_TIMER:
9718 case VMX_EXIT_IO_INSTR:
9719 break;
9720
9721 /* Errors and unexpected events. */
9722 case VMX_EXIT_INIT_SIGNAL:
9723 case VMX_EXIT_SIPI:
9724 case VMX_EXIT_IO_SMI:
9725 case VMX_EXIT_SMI:
9726 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9727 case VMX_EXIT_ERR_MSR_LOAD:
9728 case VMX_EXIT_ERR_MACHINE_CHECK:
9729 break;
9730
9731 default:
9732 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9733 break;
9734 }
9735#undef SET_BOTH
9736#undef SET_EXIT
9737
9738 /*
9739 * Dtrace tracepoints go first. We do them here at once so we don't
9740 * have to copy the guest state saving and stuff a few dozen times.
9741 * Down side is that we've got to repeat the switch, though this time
9742 * we use enmEvent since the probes are a subset of what DBGF does.
9743 */
9744 if (fDtrace1 || fDtrace2)
9745 {
9746 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9747 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9748 switch (enmEvent1)
9749 {
9750 /** @todo consider which extra parameters would be helpful for each probe. */
9751 case DBGFEVENT_END: break;
9752 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9753 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9754 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9755 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9756 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9757 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9758 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9759 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9760 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9761 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9762 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9763 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9764 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9765 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9766 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9767 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9768 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9769 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9770 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9771 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9772 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9773 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9774 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9775 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9776 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9777 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9778 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9779 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9780 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9781 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9782 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9783 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9784 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9785 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9786 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9787 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9788 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9789 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9790 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9791 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9792 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9793 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9794 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9795 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9796 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9797 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9798 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9799 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9800 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9801 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9802 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9803 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9804 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9805 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9806 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9807 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9808 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9809 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9810 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9811 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9812 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9813 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9814 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9815 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9816 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9817 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9818 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9819 }
9820 switch (enmEvent2)
9821 {
9822 /** @todo consider which extra parameters would be helpful for each probe. */
9823 case DBGFEVENT_END: break;
9824 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9825 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9826 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9827 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9828 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9829 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9830 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9831 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9832 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9833 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9834 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9835 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9836 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9837 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9838 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9839 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9840 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9841 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9842 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9843 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9844 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9845 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9846 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9847 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9848 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9849 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9850 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9852 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9853 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9854 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9855 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9856 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9857 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9858 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9866 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9867 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9868 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9871 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9872 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9874 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9875 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9876 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9877 }
9878 }
9879
9880 /*
9881 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9882 * the DBGF call will do a full check).
9883 *
9884 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9885 * Note! If we have to events, we prioritize the first, i.e. the instruction
9886 * one, in order to avoid event nesting.
9887 */
9888 if ( enmEvent1 != DBGFEVENT_END
9889 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9890 {
9891 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9892 if (rcStrict != VINF_SUCCESS)
9893 return rcStrict;
9894 }
9895 else if ( enmEvent2 != DBGFEVENT_END
9896 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9897 {
9898 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9899 if (rcStrict != VINF_SUCCESS)
9900 return rcStrict;
9901 }
9902
9903 return VINF_SUCCESS;
9904}
9905
9906
9907/**
9908 * Single-stepping VM-exit filtering.
9909 *
9910 * This is preprocessing the exits and deciding whether we've gotten far enough
9911 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9912 * performed.
9913 *
9914 * @returns Strict VBox status code (i.e. informational status codes too).
9915 * @param pVM The cross context VM structure.
9916 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9917 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9918 * out-of-sync. Make sure to update the required
9919 * fields before using them.
9920 * @param pVmxTransient Pointer to the VMX-transient structure.
9921 * @param uExitReason The VM-exit reason.
9922 * @param pDbgState The debug state.
9923 */
9924DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9925 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9926{
9927 /*
9928 * Expensive (saves context) generic dtrace exit probe.
9929 */
9930 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9931 { /* more likely */ }
9932 else
9933 {
9934 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9935 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9936 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9937 }
9938
9939 /*
9940 * Check for host NMI, just to get that out of the way.
9941 */
9942 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9943 { /* normally likely */ }
9944 else
9945 {
9946 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9947 AssertRCReturn(rc2, rc2);
9948 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9949 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9950 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9951 }
9952
9953 /*
9954 * Check for single stepping event if we're stepping.
9955 */
9956 if (pVCpu->hm.s.fSingleInstruction)
9957 {
9958 switch (uExitReason)
9959 {
9960 case VMX_EXIT_MTF:
9961 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9962
9963 /* Various events: */
9964 case VMX_EXIT_XCPT_OR_NMI:
9965 case VMX_EXIT_EXT_INT:
9966 case VMX_EXIT_TRIPLE_FAULT:
9967 case VMX_EXIT_INT_WINDOW:
9968 case VMX_EXIT_NMI_WINDOW:
9969 case VMX_EXIT_TASK_SWITCH:
9970 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9971 case VMX_EXIT_APIC_ACCESS:
9972 case VMX_EXIT_EPT_VIOLATION:
9973 case VMX_EXIT_EPT_MISCONFIG:
9974 case VMX_EXIT_PREEMPT_TIMER:
9975
9976 /* Instruction specific VM-exits: */
9977 case VMX_EXIT_CPUID:
9978 case VMX_EXIT_GETSEC:
9979 case VMX_EXIT_HLT:
9980 case VMX_EXIT_INVD:
9981 case VMX_EXIT_INVLPG:
9982 case VMX_EXIT_RDPMC:
9983 case VMX_EXIT_RDTSC:
9984 case VMX_EXIT_RSM:
9985 case VMX_EXIT_VMCALL:
9986 case VMX_EXIT_VMCLEAR:
9987 case VMX_EXIT_VMLAUNCH:
9988 case VMX_EXIT_VMPTRLD:
9989 case VMX_EXIT_VMPTRST:
9990 case VMX_EXIT_VMREAD:
9991 case VMX_EXIT_VMRESUME:
9992 case VMX_EXIT_VMWRITE:
9993 case VMX_EXIT_VMXOFF:
9994 case VMX_EXIT_VMXON:
9995 case VMX_EXIT_MOV_CRX:
9996 case VMX_EXIT_MOV_DRX:
9997 case VMX_EXIT_IO_INSTR:
9998 case VMX_EXIT_RDMSR:
9999 case VMX_EXIT_WRMSR:
10000 case VMX_EXIT_MWAIT:
10001 case VMX_EXIT_MONITOR:
10002 case VMX_EXIT_PAUSE:
10003 case VMX_EXIT_XDTR_ACCESS:
10004 case VMX_EXIT_TR_ACCESS:
10005 case VMX_EXIT_INVEPT:
10006 case VMX_EXIT_RDTSCP:
10007 case VMX_EXIT_INVVPID:
10008 case VMX_EXIT_WBINVD:
10009 case VMX_EXIT_XSETBV:
10010 case VMX_EXIT_RDRAND:
10011 case VMX_EXIT_INVPCID:
10012 case VMX_EXIT_VMFUNC:
10013 case VMX_EXIT_RDSEED:
10014 case VMX_EXIT_XSAVES:
10015 case VMX_EXIT_XRSTORS:
10016 {
10017 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10018 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10019 AssertRCReturn(rc2, rc2);
10020 if ( pMixedCtx->rip != pDbgState->uRipStart
10021 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10022 return VINF_EM_DBG_STEPPED;
10023 break;
10024 }
10025
10026 /* Errors and unexpected events: */
10027 case VMX_EXIT_INIT_SIGNAL:
10028 case VMX_EXIT_SIPI:
10029 case VMX_EXIT_IO_SMI:
10030 case VMX_EXIT_SMI:
10031 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10032 case VMX_EXIT_ERR_MSR_LOAD:
10033 case VMX_EXIT_ERR_MACHINE_CHECK:
10034 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10035 break;
10036
10037 default:
10038 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
10039 break;
10040 }
10041 }
10042
10043 /*
10044 * Check for debugger event breakpoints and dtrace probes.
10045 */
10046 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10047 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10048 {
10049 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10050 if (rcStrict != VINF_SUCCESS)
10051 return rcStrict;
10052 }
10053
10054 /*
10055 * Normal processing.
10056 */
10057#ifdef HMVMX_USE_FUNCTION_TABLE
10058 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10059#else
10060 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10061#endif
10062}
10063
10064
10065/**
10066 * Single steps guest code using VT-x.
10067 *
10068 * @returns Strict VBox status code (i.e. informational status codes too).
10069 * @param pVM The cross context VM structure.
10070 * @param pVCpu The cross context virtual CPU structure.
10071 * @param pCtx Pointer to the guest-CPU context.
10072 *
10073 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10074 */
10075static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10076{
10077 VMXTRANSIENT VmxTransient;
10078 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10079
10080 /* Set HMCPU indicators. */
10081 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10082 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10083 pVCpu->hm.s.fDebugWantRdTscExit = false;
10084 pVCpu->hm.s.fUsingDebugLoop = true;
10085
10086 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10087 VMXRUNDBGSTATE DbgState;
10088 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10089 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10090
10091 /*
10092 * The loop.
10093 */
10094 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10095 for (uint32_t cLoops = 0; ; cLoops++)
10096 {
10097 Assert(!HMR0SuspendPending());
10098 HMVMX_ASSERT_CPU_SAFE();
10099 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10100
10101 /*
10102 * Preparatory work for running guest code, this may force us to return
10103 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10104 */
10105 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10106 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10107 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10108 if (rcStrict != VINF_SUCCESS)
10109 break;
10110
10111 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10112 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10113
10114 /*
10115 * Now we can run the guest code.
10116 */
10117 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10118
10119 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10120
10121 /*
10122 * Restore any residual host-state and save any bits shared between host
10123 * and guest into the guest-CPU state. Re-enables interrupts!
10124 */
10125 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10126
10127 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10128 if (RT_SUCCESS(rcRun))
10129 { /* very likely */ }
10130 else
10131 {
10132 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10133 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10134 return rcRun;
10135 }
10136
10137 /* Profile the VM-exit. */
10138 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10139 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10140 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10141 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10142 HMVMX_START_EXIT_DISPATCH_PROF();
10143
10144 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10145
10146 /*
10147 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10148 */
10149 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10150 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10151 if (rcStrict != VINF_SUCCESS)
10152 break;
10153 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10154 {
10155 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10156 rcStrict = VINF_EM_RAW_INTERRUPT;
10157 break;
10158 }
10159
10160 /*
10161 * Stepping: Did the RIP change, if so, consider it a single step.
10162 * Otherwise, make sure one of the TFs gets set.
10163 */
10164 if (fStepping)
10165 {
10166 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10167 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10168 AssertRCReturn(rc2, rc2);
10169 if ( pCtx->rip != DbgState.uRipStart
10170 || pCtx->cs.Sel != DbgState.uCsStart)
10171 {
10172 rcStrict = VINF_EM_DBG_STEPPED;
10173 break;
10174 }
10175 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10176 }
10177
10178 /*
10179 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10180 */
10181 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10182 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10183 }
10184
10185 /*
10186 * Clear the X86_EFL_TF if necessary.
10187 */
10188 if (pVCpu->hm.s.fClearTrapFlag)
10189 {
10190 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10191 AssertRCReturn(rc2, rc2);
10192 pVCpu->hm.s.fClearTrapFlag = false;
10193 pCtx->eflags.Bits.u1TF = 0;
10194 }
10195 /** @todo there seems to be issues with the resume flag when the monitor trap
10196 * flag is pending without being used. Seen early in bios init when
10197 * accessing APIC page in protected mode. */
10198
10199 /*
10200 * Restore VM-exit control settings as we may not reenter this function the
10201 * next time around.
10202 */
10203 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10204
10205 /* Restore HMCPU indicators. */
10206 pVCpu->hm.s.fUsingDebugLoop = false;
10207 pVCpu->hm.s.fDebugWantRdTscExit = false;
10208 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10209
10210 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10211 return rcStrict;
10212}
10213
10214
10215/** @} */
10216
10217
10218/**
10219 * Checks if any expensive dtrace probes are enabled and we should go to the
10220 * debug loop.
10221 *
10222 * @returns true if we should use debug loop, false if not.
10223 */
10224static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10225{
10226 /* It's probably faster to OR the raw 32-bit counter variables together.
10227 Since the variables are in an array and the probes are next to one
10228 another (more or less), we have good locality. So, better read
10229 eight-nine cache lines ever time and only have one conditional, than
10230 128+ conditionals, right? */
10231 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10232 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10233 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10234 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10235 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10236 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10237 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10238 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10239 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10240 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10241 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10242 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10243 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10244 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10245 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10246 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10247 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10248 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10249 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10250 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10251 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10252 ) != 0
10253 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10254 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10255 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10256 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10257 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10258 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10259 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10260 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10261 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10262 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10263 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10264 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10265 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10266 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10267 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10268 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10269 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10270 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10271 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10272 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10273 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10274 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10275 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10276 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10277 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10278 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10279 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10280 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10281 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10282 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10283 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10284 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10285 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10286 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10287 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10288 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10289 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10290 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10291 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10292 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10293 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10294 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10295 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10296 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10297 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10298 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10299 ) != 0
10300 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10301 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10302 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10303 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10304 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10305 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10306 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10307 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10308 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10309 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10310 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10311 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10312 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10313 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10314 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10315 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10316 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10317 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10318 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10319 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10320 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10321 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10322 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10323 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10324 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10325 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10326 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10327 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10328 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10329 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10330 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10331 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10332 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10333 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10334 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10335 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10336 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10337 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10338 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10339 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10340 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10341 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10342 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10343 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10344 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10345 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10346 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10347 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10348 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10349 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10350 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10351 ) != 0;
10352}
10353
10354
10355/**
10356 * Runs the guest code using VT-x.
10357 *
10358 * @returns Strict VBox status code (i.e. informational status codes too).
10359 * @param pVM The cross context VM structure.
10360 * @param pVCpu The cross context virtual CPU structure.
10361 * @param pCtx Pointer to the guest-CPU context.
10362 */
10363VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10364{
10365 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10366 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10367 HMVMX_ASSERT_PREEMPT_SAFE();
10368
10369 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10370
10371 VBOXSTRICTRC rcStrict;
10372 if ( !pVCpu->hm.s.fUseDebugLoop
10373 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10374 && !DBGFIsStepping(pVCpu) )
10375 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10376 else
10377 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10378
10379 if (rcStrict == VERR_EM_INTERPRETER)
10380 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10381 else if (rcStrict == VINF_EM_RESET)
10382 rcStrict = VINF_EM_TRIPLE_FAULT;
10383
10384 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10385 if (RT_FAILURE(rc2))
10386 {
10387 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10388 rcStrict = rc2;
10389 }
10390 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10391 return rcStrict;
10392}
10393
10394
10395#ifndef HMVMX_USE_FUNCTION_TABLE
10396DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10397{
10398# ifdef DEBUG_ramshankar
10399# define RETURN_EXIT_CALL(a_CallExpr) \
10400 do { \
10401 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10402 VBOXSTRICTRC rcStrict = a_CallExpr; \
10403 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10404 return rcStrict; \
10405 } while (0)
10406# else
10407# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10408# endif
10409 switch (rcReason)
10410 {
10411 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10412 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10413 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10414 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10415 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10416 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10417 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10418 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10419 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10420 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10421 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10422 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10423 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10424 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10425 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10426 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10427 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10428 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10429 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10430 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10431 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10432 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10433 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10434 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10435 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10436 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10437 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10438 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10439 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10440 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10441 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10442 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10443 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10444 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10445
10446 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10447 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10448 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10449 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10450 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10451 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10452 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10453 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10454 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10455
10456 case VMX_EXIT_VMCLEAR:
10457 case VMX_EXIT_VMLAUNCH:
10458 case VMX_EXIT_VMPTRLD:
10459 case VMX_EXIT_VMPTRST:
10460 case VMX_EXIT_VMREAD:
10461 case VMX_EXIT_VMRESUME:
10462 case VMX_EXIT_VMWRITE:
10463 case VMX_EXIT_VMXOFF:
10464 case VMX_EXIT_VMXON:
10465 case VMX_EXIT_INVEPT:
10466 case VMX_EXIT_INVVPID:
10467 case VMX_EXIT_VMFUNC:
10468 case VMX_EXIT_XSAVES:
10469 case VMX_EXIT_XRSTORS:
10470 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10471 case VMX_EXIT_RESERVED_60:
10472 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10473 case VMX_EXIT_RESERVED_62:
10474 default:
10475 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10476 }
10477#undef RETURN_EXIT_CALL
10478}
10479#endif /* !HMVMX_USE_FUNCTION_TABLE */
10480
10481
10482#ifdef VBOX_STRICT
10483/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10484# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10485 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10486
10487# define HMVMX_ASSERT_PREEMPT_CPUID() \
10488 do { \
10489 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10490 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10491 } while (0)
10492
10493# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10494 do { \
10495 AssertPtr(pVCpu); \
10496 AssertPtr(pMixedCtx); \
10497 AssertPtr(pVmxTransient); \
10498 Assert(pVmxTransient->fVMEntryFailed == false); \
10499 Assert(ASMIntAreEnabled()); \
10500 HMVMX_ASSERT_PREEMPT_SAFE(); \
10501 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10502 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10503 HMVMX_ASSERT_PREEMPT_SAFE(); \
10504 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10505 HMVMX_ASSERT_PREEMPT_CPUID(); \
10506 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10507 } while (0)
10508
10509# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10510 do { \
10511 Log4Func(("\n")); \
10512 } while (0)
10513#else /* nonstrict builds: */
10514# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10515 do { \
10516 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10517 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10518 } while (0)
10519# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10520#endif
10521
10522
10523/**
10524 * Advances the guest RIP by the specified number of bytes.
10525 *
10526 * @param pVCpu The cross context virtual CPU structure.
10527 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10528 * out-of-sync. Make sure to update the required fields
10529 * before using them.
10530 * @param cbInstr Number of bytes to advance the RIP by.
10531 *
10532 * @remarks No-long-jump zone!!!
10533 */
10534DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10535{
10536 /* Advance the RIP. */
10537 pMixedCtx->rip += cbInstr;
10538 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10539
10540 /* Update interrupt inhibition. */
10541 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10542 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10543 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10544}
10545
10546
10547/**
10548 * Advances the guest RIP after reading it from the VMCS.
10549 *
10550 * @returns VBox status code, no informational status codes.
10551 * @param pVCpu The cross context virtual CPU structure.
10552 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10553 * out-of-sync. Make sure to update the required fields
10554 * before using them.
10555 * @param pVmxTransient Pointer to the VMX transient structure.
10556 *
10557 * @remarks No-long-jump zone!!!
10558 */
10559static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10560{
10561 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10562 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10563 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10564 AssertRCReturn(rc, rc);
10565
10566 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10567
10568 /*
10569 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10570 * pending debug exception field as it takes care of priority of events.
10571 *
10572 * See Intel spec. 32.2.1 "Debug Exceptions".
10573 */
10574 if ( !pVCpu->hm.s.fSingleInstruction
10575 && pMixedCtx->eflags.Bits.u1TF)
10576 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10577
10578 return VINF_SUCCESS;
10579}
10580
10581
10582/**
10583 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10584 * and update error record fields accordingly.
10585 *
10586 * @return VMX_IGS_* return codes.
10587 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10588 * wrong with the guest state.
10589 *
10590 * @param pVM The cross context VM structure.
10591 * @param pVCpu The cross context virtual CPU structure.
10592 * @param pCtx Pointer to the guest-CPU state.
10593 *
10594 * @remarks This function assumes our cache of the VMCS controls
10595 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10596 */
10597static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10598{
10599#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10600#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10601 uError = (err); \
10602 break; \
10603 } else do { } while (0)
10604
10605 int rc;
10606 uint32_t uError = VMX_IGS_ERROR;
10607 uint32_t u32Val;
10608 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10609
10610 do
10611 {
10612 /*
10613 * CR0.
10614 */
10615 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10616 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10617 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10618 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10619 if (fUnrestrictedGuest)
10620 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10621
10622 uint32_t u32GuestCR0;
10623 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10624 AssertRCBreak(rc);
10625 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10626 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10627 if ( !fUnrestrictedGuest
10628 && (u32GuestCR0 & X86_CR0_PG)
10629 && !(u32GuestCR0 & X86_CR0_PE))
10630 {
10631 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10632 }
10633
10634 /*
10635 * CR4.
10636 */
10637 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10638 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10639
10640 uint32_t u32GuestCR4;
10641 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10642 AssertRCBreak(rc);
10643 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10644 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10645
10646 /*
10647 * IA32_DEBUGCTL MSR.
10648 */
10649 uint64_t u64Val;
10650 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10651 AssertRCBreak(rc);
10652 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10653 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10654 {
10655 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10656 }
10657 uint64_t u64DebugCtlMsr = u64Val;
10658
10659#ifdef VBOX_STRICT
10660 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10661 AssertRCBreak(rc);
10662 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10663#endif
10664 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10665
10666 /*
10667 * RIP and RFLAGS.
10668 */
10669 uint32_t u32Eflags;
10670#if HC_ARCH_BITS == 64
10671 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10672 AssertRCBreak(rc);
10673 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10674 if ( !fLongModeGuest
10675 || !pCtx->cs.Attr.n.u1Long)
10676 {
10677 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10678 }
10679 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10680 * must be identical if the "IA-32e mode guest" VM-entry
10681 * control is 1 and CS.L is 1. No check applies if the
10682 * CPU supports 64 linear-address bits. */
10683
10684 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10685 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10686 AssertRCBreak(rc);
10687 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10688 VMX_IGS_RFLAGS_RESERVED);
10689 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10690 u32Eflags = u64Val;
10691#else
10692 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10693 AssertRCBreak(rc);
10694 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10695 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10696#endif
10697
10698 if ( fLongModeGuest
10699 || ( fUnrestrictedGuest
10700 && !(u32GuestCR0 & X86_CR0_PE)))
10701 {
10702 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10703 }
10704
10705 uint32_t u32EntryInfo;
10706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10707 AssertRCBreak(rc);
10708 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10709 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10710 {
10711 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10712 }
10713
10714 /*
10715 * 64-bit checks.
10716 */
10717#if HC_ARCH_BITS == 64
10718 if (fLongModeGuest)
10719 {
10720 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10721 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10722 }
10723
10724 if ( !fLongModeGuest
10725 && (u32GuestCR4 & X86_CR4_PCIDE))
10726 {
10727 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10728 }
10729
10730 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10731 * 51:32 beyond the processor's physical-address width are 0. */
10732
10733 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10734 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10735 {
10736 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10737 }
10738
10739 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10740 AssertRCBreak(rc);
10741 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10742
10743 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10744 AssertRCBreak(rc);
10745 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10746#endif
10747
10748 /*
10749 * PERF_GLOBAL MSR.
10750 */
10751 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10752 {
10753 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10754 AssertRCBreak(rc);
10755 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10756 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10757 }
10758
10759 /*
10760 * PAT MSR.
10761 */
10762 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10763 {
10764 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10765 AssertRCBreak(rc);
10766 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10767 for (unsigned i = 0; i < 8; i++)
10768 {
10769 uint8_t u8Val = (u64Val & 0xff);
10770 if ( u8Val != 0 /* UC */
10771 && u8Val != 1 /* WC */
10772 && u8Val != 4 /* WT */
10773 && u8Val != 5 /* WP */
10774 && u8Val != 6 /* WB */
10775 && u8Val != 7 /* UC- */)
10776 {
10777 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10778 }
10779 u64Val >>= 8;
10780 }
10781 }
10782
10783 /*
10784 * EFER MSR.
10785 */
10786 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10787 {
10788 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10789 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10790 AssertRCBreak(rc);
10791 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10792 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10793 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10794 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10795 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10796 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10797 || !(u32GuestCR0 & X86_CR0_PG)
10798 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10799 VMX_IGS_EFER_LMA_LME_MISMATCH);
10800 }
10801
10802 /*
10803 * Segment registers.
10804 */
10805 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10806 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10807 if (!(u32Eflags & X86_EFL_VM))
10808 {
10809 /* CS */
10810 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10811 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10812 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10813 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10814 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10815 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10816 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10817 /* CS cannot be loaded with NULL in protected mode. */
10818 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10819 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10820 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10821 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10822 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10823 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10824 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10825 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10826 else
10827 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10828
10829 /* SS */
10830 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10831 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10832 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10833 if ( !(pCtx->cr0 & X86_CR0_PE)
10834 || pCtx->cs.Attr.n.u4Type == 3)
10835 {
10836 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10837 }
10838 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10839 {
10840 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10841 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10842 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10843 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10844 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10845 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10846 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10847 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10848 }
10849
10850 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10851 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10852 {
10853 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10854 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10855 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10856 || pCtx->ds.Attr.n.u4Type > 11
10857 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10858 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10859 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10860 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10861 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10862 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10863 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10864 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10865 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10866 }
10867 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10868 {
10869 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10870 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10871 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10872 || pCtx->es.Attr.n.u4Type > 11
10873 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10874 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10875 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10876 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10877 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10878 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10879 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10880 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10881 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10882 }
10883 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10884 {
10885 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10886 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10887 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10888 || pCtx->fs.Attr.n.u4Type > 11
10889 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10890 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10891 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10892 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10893 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10894 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10895 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10896 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10897 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10898 }
10899 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10900 {
10901 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10902 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10903 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10904 || pCtx->gs.Attr.n.u4Type > 11
10905 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10906 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10907 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10908 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10909 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10910 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10911 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10912 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10913 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10914 }
10915 /* 64-bit capable CPUs. */
10916#if HC_ARCH_BITS == 64
10917 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10918 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10919 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10920 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10921 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10922 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10923 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10924 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10925 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10926 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10927 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10928#endif
10929 }
10930 else
10931 {
10932 /* V86 mode checks. */
10933 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10934 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10935 {
10936 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10937 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10938 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10939 }
10940 else
10941 {
10942 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10943 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10944 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10945 }
10946
10947 /* CS */
10948 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10949 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10950 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10951 /* SS */
10952 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10953 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10954 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10955 /* DS */
10956 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10957 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10958 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10959 /* ES */
10960 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10961 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10962 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10963 /* FS */
10964 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10965 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10966 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10967 /* GS */
10968 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10969 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10970 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10971 /* 64-bit capable CPUs. */
10972#if HC_ARCH_BITS == 64
10973 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10974 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10975 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10976 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10977 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10978 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10979 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10980 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10981 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10982 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10983 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10984#endif
10985 }
10986
10987 /*
10988 * TR.
10989 */
10990 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10991 /* 64-bit capable CPUs. */
10992#if HC_ARCH_BITS == 64
10993 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10994#endif
10995 if (fLongModeGuest)
10996 {
10997 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10998 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10999 }
11000 else
11001 {
11002 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11003 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11004 VMX_IGS_TR_ATTR_TYPE_INVALID);
11005 }
11006 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11007 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11008 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11009 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11010 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11011 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11012 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11013 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11014
11015 /*
11016 * GDTR and IDTR.
11017 */
11018#if HC_ARCH_BITS == 64
11019 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11020 AssertRCBreak(rc);
11021 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11022
11023 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11024 AssertRCBreak(rc);
11025 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11026#endif
11027
11028 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11029 AssertRCBreak(rc);
11030 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11031
11032 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11033 AssertRCBreak(rc);
11034 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11035
11036 /*
11037 * Guest Non-Register State.
11038 */
11039 /* Activity State. */
11040 uint32_t u32ActivityState;
11041 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11042 AssertRCBreak(rc);
11043 HMVMX_CHECK_BREAK( !u32ActivityState
11044 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11045 VMX_IGS_ACTIVITY_STATE_INVALID);
11046 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11047 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11048 uint32_t u32IntrState;
11049 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11050 AssertRCBreak(rc);
11051 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11052 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11053 {
11054 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11055 }
11056
11057 /** @todo Activity state and injecting interrupts. Left as a todo since we
11058 * currently don't use activity states but ACTIVE. */
11059
11060 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11061 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11062
11063 /* Guest interruptibility-state. */
11064 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11065 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11066 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11067 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11068 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11069 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11070 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11071 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11072 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11073 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11074 {
11075 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11076 {
11077 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11078 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11079 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11080 }
11081 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11082 {
11083 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11084 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11085 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11086 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11087 }
11088 }
11089 /** @todo Assumes the processor is not in SMM. */
11090 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11091 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11092 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11093 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11094 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11095 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11096 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11097 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11098 {
11099 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11100 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11101 }
11102
11103 /* Pending debug exceptions. */
11104#if HC_ARCH_BITS == 64
11105 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11106 AssertRCBreak(rc);
11107 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11108 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11109 u32Val = u64Val; /* For pending debug exceptions checks below. */
11110#else
11111 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11112 AssertRCBreak(rc);
11113 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11114 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11115#endif
11116
11117 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11118 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11119 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11120 {
11121 if ( (u32Eflags & X86_EFL_TF)
11122 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11123 {
11124 /* Bit 14 is PendingDebug.BS. */
11125 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11126 }
11127 if ( !(u32Eflags & X86_EFL_TF)
11128 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11129 {
11130 /* Bit 14 is PendingDebug.BS. */
11131 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11132 }
11133 }
11134
11135 /* VMCS link pointer. */
11136 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11137 AssertRCBreak(rc);
11138 if (u64Val != UINT64_C(0xffffffffffffffff))
11139 {
11140 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11141 /** @todo Bits beyond the processor's physical-address width MBZ. */
11142 /** @todo 32-bit located in memory referenced by value of this field (as a
11143 * physical address) must contain the processor's VMCS revision ID. */
11144 /** @todo SMM checks. */
11145 }
11146
11147 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11148 * not using Nested Paging? */
11149 if ( pVM->hm.s.fNestedPaging
11150 && !fLongModeGuest
11151 && CPUMIsGuestInPAEModeEx(pCtx))
11152 {
11153 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11154 AssertRCBreak(rc);
11155 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11156
11157 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11158 AssertRCBreak(rc);
11159 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11160
11161 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11162 AssertRCBreak(rc);
11163 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11164
11165 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11166 AssertRCBreak(rc);
11167 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11168 }
11169
11170 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11171 if (uError == VMX_IGS_ERROR)
11172 uError = VMX_IGS_REASON_NOT_FOUND;
11173 } while (0);
11174
11175 pVCpu->hm.s.u32HMError = uError;
11176 return uError;
11177
11178#undef HMVMX_ERROR_BREAK
11179#undef HMVMX_CHECK_BREAK
11180}
11181
11182/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11183/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11184/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11185
11186/** @name VM-exit handlers.
11187 * @{
11188 */
11189
11190/**
11191 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11192 */
11193HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11194{
11195 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11196 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11197 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11198 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11199 return VINF_SUCCESS;
11200 return VINF_EM_RAW_INTERRUPT;
11201}
11202
11203
11204/**
11205 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11206 */
11207HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11208{
11209 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11210 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11211
11212 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11213 AssertRCReturn(rc, rc);
11214
11215 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11216 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11217 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11218 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11219
11220 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11221 {
11222 /*
11223 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11224 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11225 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11226 *
11227 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11228 */
11229 VMXDispatchHostNmi();
11230 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11231 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11232 return VINF_SUCCESS;
11233 }
11234
11235 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11236 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11237 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11238 { /* likely */ }
11239 else
11240 {
11241 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11242 rcStrictRc1 = VINF_SUCCESS;
11243 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11244 return rcStrictRc1;
11245 }
11246
11247 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11248 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11249 switch (uIntType)
11250 {
11251 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11252 Assert(uVector == X86_XCPT_DB);
11253 /* no break */
11254 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11255 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11256 /* no break */
11257 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11258 {
11259 /*
11260 * If there's any exception caused as a result of event injection, go back to
11261 * the interpreter. The page-fault case is complicated and we manually handle
11262 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11263 * handled in hmR0VmxCheckExitDueToEventDelivery.
11264 */
11265 if (!pVCpu->hm.s.Event.fPending)
11266 { /* likely */ }
11267 else if ( uVector != X86_XCPT_PF
11268 && uVector != X86_XCPT_AC)
11269 {
11270 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11271 rc = VERR_EM_INTERPRETER;
11272 break;
11273 }
11274
11275 switch (uVector)
11276 {
11277 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11278 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11279 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11280 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11281 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11282 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11283 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11284
11285 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11286 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11287 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11288 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11289 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11290 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11291 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11292 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11293 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11294 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11295 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11296 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11297 default:
11298 {
11299 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11300 AssertRCReturn(rc, rc);
11301
11302 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11303 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11304 {
11305 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11306 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11307 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11308
11309 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11310 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11311 AssertRCReturn(rc, rc);
11312 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11313 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11314 0 /* GCPtrFaultAddress */);
11315 AssertRCReturn(rc, rc);
11316 }
11317 else
11318 {
11319 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11320 pVCpu->hm.s.u32HMError = uVector;
11321 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11322 }
11323 break;
11324 }
11325 }
11326 break;
11327 }
11328
11329 default:
11330 {
11331 pVCpu->hm.s.u32HMError = uExitIntInfo;
11332 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11333 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11334 break;
11335 }
11336 }
11337 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11338 return rc;
11339}
11340
11341
11342/**
11343 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11344 */
11345HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11346{
11347 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11348
11349 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11350 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11351
11352 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11353 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11354 return VINF_SUCCESS;
11355}
11356
11357
11358/**
11359 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11360 */
11361HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11362{
11363 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11364 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11365 {
11366 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11367 HMVMX_RETURN_UNEXPECTED_EXIT();
11368 }
11369
11370 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11371
11372 /*
11373 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11374 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11375 */
11376 uint32_t uIntrState = 0;
11377 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11378 AssertRCReturn(rc, rc);
11379
11380 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11381 if ( fBlockSti
11382 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11383 {
11384 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11385 }
11386
11387 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11388 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11389
11390 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11391 return VINF_SUCCESS;
11392}
11393
11394
11395/**
11396 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11397 */
11398HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11399{
11400 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11401 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11402 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11403}
11404
11405
11406/**
11407 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11408 */
11409HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11410{
11411 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11412 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11413 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11414}
11415
11416
11417/**
11418 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11419 */
11420HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11421{
11422 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11423 PVM pVM = pVCpu->CTX_SUFF(pVM);
11424 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11425 if (RT_LIKELY(rc == VINF_SUCCESS))
11426 {
11427 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11428 Assert(pVmxTransient->cbInstr == 2);
11429 }
11430 else
11431 {
11432 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11433 rc = VERR_EM_INTERPRETER;
11434 }
11435 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11436 return rc;
11437}
11438
11439
11440/**
11441 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11442 */
11443HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11444{
11445 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11446 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11447 AssertRCReturn(rc, rc);
11448
11449 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11450 return VINF_EM_RAW_EMULATE_INSTR;
11451
11452 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11453 HMVMX_RETURN_UNEXPECTED_EXIT();
11454}
11455
11456
11457/**
11458 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11459 */
11460HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11461{
11462 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11463 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11464 AssertRCReturn(rc, rc);
11465
11466 PVM pVM = pVCpu->CTX_SUFF(pVM);
11467 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11468 if (RT_LIKELY(rc == VINF_SUCCESS))
11469 {
11470 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11471 Assert(pVmxTransient->cbInstr == 2);
11472 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11473 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11474 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11475 }
11476 else
11477 rc = VERR_EM_INTERPRETER;
11478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11479 return rc;
11480}
11481
11482
11483/**
11484 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11485 */
11486HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11487{
11488 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11489 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11490 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11491 AssertRCReturn(rc, rc);
11492
11493 PVM pVM = pVCpu->CTX_SUFF(pVM);
11494 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11495 if (RT_SUCCESS(rc))
11496 {
11497 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11498 Assert(pVmxTransient->cbInstr == 3);
11499 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11500 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11501 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11502 }
11503 else
11504 {
11505 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11506 rc = VERR_EM_INTERPRETER;
11507 }
11508 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11509 return rc;
11510}
11511
11512
11513/**
11514 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11515 */
11516HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11517{
11518 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11519 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11520 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11521 AssertRCReturn(rc, rc);
11522
11523 PVM pVM = pVCpu->CTX_SUFF(pVM);
11524 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11525 if (RT_LIKELY(rc == VINF_SUCCESS))
11526 {
11527 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11528 Assert(pVmxTransient->cbInstr == 2);
11529 }
11530 else
11531 {
11532 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11533 rc = VERR_EM_INTERPRETER;
11534 }
11535 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11536 return rc;
11537}
11538
11539
11540/**
11541 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11542 */
11543HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11544{
11545 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11546 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11547
11548 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11549 if (pVCpu->hm.s.fHypercallsEnabled)
11550 {
11551#if 0
11552 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11553#else
11554 /* Aggressive state sync. for now. */
11555 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11556 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11557 AssertRCReturn(rc, rc);
11558#endif
11559
11560 /* Perform the hypercall. */
11561 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11562 if (rcStrict == VINF_SUCCESS)
11563 {
11564 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11565 AssertRCReturn(rc, rc);
11566 }
11567 else
11568 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11569 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11570 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11571
11572 /* If the hypercall changes anything other than guest's general-purpose registers,
11573 we would need to reload the guest changed bits here before VM-entry. */
11574 }
11575 else
11576 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11577
11578 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11579 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11580 {
11581 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11582 rcStrict = VINF_SUCCESS;
11583 }
11584
11585 return rcStrict;
11586}
11587
11588
11589/**
11590 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11591 */
11592HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11593{
11594 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11595 PVM pVM = pVCpu->CTX_SUFF(pVM);
11596 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11597
11598 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11599 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11600 AssertRCReturn(rc, rc);
11601
11602 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11603 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11604 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11605 else
11606 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11607 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11608 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11609 return rcStrict;
11610}
11611
11612
11613/**
11614 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11615 */
11616HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11617{
11618 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11619 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11620 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11621 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11622 AssertRCReturn(rc, rc);
11623
11624 PVM pVM = pVCpu->CTX_SUFF(pVM);
11625 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11626 if (RT_LIKELY(rc == VINF_SUCCESS))
11627 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11628 else
11629 {
11630 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11631 rc = VERR_EM_INTERPRETER;
11632 }
11633 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11634 return rc;
11635}
11636
11637
11638/**
11639 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11640 */
11641HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11642{
11643 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11644 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11645 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11646 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11647 AssertRCReturn(rc, rc);
11648
11649 PVM pVM = pVCpu->CTX_SUFF(pVM);
11650 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11651 rc = VBOXSTRICTRC_VAL(rc2);
11652 if (RT_LIKELY( rc == VINF_SUCCESS
11653 || rc == VINF_EM_HALT))
11654 {
11655 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11656 AssertRCReturn(rc3, rc3);
11657
11658 if ( rc == VINF_EM_HALT
11659 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11660 {
11661 rc = VINF_SUCCESS;
11662 }
11663 }
11664 else
11665 {
11666 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11667 rc = VERR_EM_INTERPRETER;
11668 }
11669 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11670 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11671 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11672 return rc;
11673}
11674
11675
11676/**
11677 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11678 */
11679HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11680{
11681 /*
11682 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11683 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11684 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11685 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11686 */
11687 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11688 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11689 HMVMX_RETURN_UNEXPECTED_EXIT();
11690}
11691
11692
11693/**
11694 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11695 */
11696HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11697{
11698 /*
11699 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11700 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11701 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11702 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11703 */
11704 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11705 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11706 HMVMX_RETURN_UNEXPECTED_EXIT();
11707}
11708
11709
11710/**
11711 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11712 */
11713HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11714{
11715 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11716 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11717 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11718 HMVMX_RETURN_UNEXPECTED_EXIT();
11719}
11720
11721
11722/**
11723 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11724 */
11725HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11726{
11727 /*
11728 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11729 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11730 * See Intel spec. 25.3 "Other Causes of VM-exits".
11731 */
11732 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11733 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11734 HMVMX_RETURN_UNEXPECTED_EXIT();
11735}
11736
11737
11738/**
11739 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11740 * VM-exit.
11741 */
11742HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11743{
11744 /*
11745 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11746 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11747 *
11748 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11749 * See Intel spec. "23.8 Restrictions on VMX operation".
11750 */
11751 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11752 return VINF_SUCCESS;
11753}
11754
11755
11756/**
11757 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11758 * VM-exit.
11759 */
11760HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11761{
11762 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11763 return VINF_EM_RESET;
11764}
11765
11766
11767/**
11768 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11769 */
11770HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11771{
11772 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11773 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11774
11775 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11776 AssertRCReturn(rc, rc);
11777
11778 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11779 rc = VINF_SUCCESS;
11780 else
11781 rc = VINF_EM_HALT;
11782
11783 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11784 if (rc != VINF_SUCCESS)
11785 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11786 return rc;
11787}
11788
11789
11790/**
11791 * VM-exit handler for instructions that result in a \#UD exception delivered to
11792 * the guest.
11793 */
11794HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11795{
11796 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11797 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11798 return VINF_SUCCESS;
11799}
11800
11801
11802/**
11803 * VM-exit handler for expiry of the VMX preemption timer.
11804 */
11805HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11806{
11807 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11808
11809 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11810 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11811
11812 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11813 PVM pVM = pVCpu->CTX_SUFF(pVM);
11814 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11815 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11816 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11817}
11818
11819
11820/**
11821 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11822 */
11823HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11824{
11825 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11826
11827 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11828 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11829 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11830 AssertRCReturn(rc, rc);
11831
11832 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11833 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11834
11835 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11836
11837 return rcStrict;
11838}
11839
11840
11841/**
11842 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11843 */
11844HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11845{
11846 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11847
11848 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11849 /** @todo implement EMInterpretInvpcid() */
11850 return VERR_EM_INTERPRETER;
11851}
11852
11853
11854/**
11855 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11856 * Error VM-exit.
11857 */
11858HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11859{
11860 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11861 AssertRCReturn(rc, rc);
11862
11863 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11864 AssertRCReturn(rc, rc);
11865
11866 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11867 NOREF(uInvalidReason);
11868
11869#ifdef VBOX_STRICT
11870 uint32_t uIntrState;
11871 RTHCUINTREG uHCReg;
11872 uint64_t u64Val;
11873 uint32_t u32Val;
11874
11875 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11876 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11877 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11878 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11879 AssertRCReturn(rc, rc);
11880
11881 Log4(("uInvalidReason %u\n", uInvalidReason));
11882 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11883 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11884 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11885 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11886
11887 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11888 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11889 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11890 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11891 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11892 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11893 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11894 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11895 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11896 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11897 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11898 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11899#else
11900 NOREF(pVmxTransient);
11901#endif
11902
11903 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11904 return VERR_VMX_INVALID_GUEST_STATE;
11905}
11906
11907
11908/**
11909 * VM-exit handler for VM-entry failure due to an MSR-load
11910 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11911 */
11912HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11913{
11914 NOREF(pVmxTransient);
11915 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11916 HMVMX_RETURN_UNEXPECTED_EXIT();
11917}
11918
11919
11920/**
11921 * VM-exit handler for VM-entry failure due to a machine-check event
11922 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11923 */
11924HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11925{
11926 NOREF(pVmxTransient);
11927 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11928 HMVMX_RETURN_UNEXPECTED_EXIT();
11929}
11930
11931
11932/**
11933 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11934 * theory.
11935 */
11936HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11937{
11938 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11939 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11940 return VERR_VMX_UNDEFINED_EXIT_CODE;
11941}
11942
11943
11944/**
11945 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11946 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11947 * Conditional VM-exit.
11948 */
11949HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11950{
11951 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11952
11953 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11955 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11956 return VERR_EM_INTERPRETER;
11957 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11958 HMVMX_RETURN_UNEXPECTED_EXIT();
11959}
11960
11961
11962/**
11963 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11964 */
11965HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11966{
11967 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11968
11969 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11971 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11972 return VERR_EM_INTERPRETER;
11973 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11974 HMVMX_RETURN_UNEXPECTED_EXIT();
11975}
11976
11977
11978/**
11979 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11980 */
11981HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11982{
11983 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11984
11985 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11986 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11987 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11988 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11989 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11990 {
11991 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11992 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11993 }
11994 AssertRCReturn(rc, rc);
11995 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11996
11997#ifdef VBOX_STRICT
11998 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11999 {
12000 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12001 && pMixedCtx->ecx != MSR_K6_EFER)
12002 {
12003 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12004 pMixedCtx->ecx));
12005 HMVMX_RETURN_UNEXPECTED_EXIT();
12006 }
12007 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12008 {
12009 VMXMSREXITREAD enmRead;
12010 VMXMSREXITWRITE enmWrite;
12011 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12012 AssertRCReturn(rc2, rc2);
12013 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12014 {
12015 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12016 HMVMX_RETURN_UNEXPECTED_EXIT();
12017 }
12018 }
12019 }
12020#endif
12021
12022 PVM pVM = pVCpu->CTX_SUFF(pVM);
12023 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12024 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12025 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12026 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12027 if (RT_SUCCESS(rc))
12028 {
12029 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12030 Assert(pVmxTransient->cbInstr == 2);
12031 }
12032 return rc;
12033}
12034
12035
12036/**
12037 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12038 */
12039HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12040{
12041 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12042 PVM pVM = pVCpu->CTX_SUFF(pVM);
12043 int rc = VINF_SUCCESS;
12044
12045 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12046 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12047 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12048 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12049 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12050 {
12051 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12052 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12053 }
12054 AssertRCReturn(rc, rc);
12055 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12056
12057 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12058 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12059 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12060
12061 if (RT_SUCCESS(rc))
12062 {
12063 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12064
12065 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12066 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12067 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
12068 {
12069 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12070 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12071 EMInterpretWrmsr() changes it. */
12072 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12073 }
12074 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12075 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12076 else if (pMixedCtx->ecx == MSR_K6_EFER)
12077 {
12078 /*
12079 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12080 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12081 * the other bits as well, SCE and NXE. See @bugref{7368}.
12082 */
12083 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12084 }
12085
12086 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12087 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12088 {
12089 switch (pMixedCtx->ecx)
12090 {
12091 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12092 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12093 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12094 case MSR_K8_FS_BASE: /* no break */
12095 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12096 case MSR_K6_EFER: /* already handled above */ break;
12097 default:
12098 {
12099 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12100 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12101 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12102 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12103 break;
12104 }
12105 }
12106 }
12107#ifdef VBOX_STRICT
12108 else
12109 {
12110 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12111 switch (pMixedCtx->ecx)
12112 {
12113 case MSR_IA32_SYSENTER_CS:
12114 case MSR_IA32_SYSENTER_EIP:
12115 case MSR_IA32_SYSENTER_ESP:
12116 case MSR_K8_FS_BASE:
12117 case MSR_K8_GS_BASE:
12118 {
12119 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12120 HMVMX_RETURN_UNEXPECTED_EXIT();
12121 }
12122
12123 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12124 default:
12125 {
12126 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12127 {
12128 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12129 if (pMixedCtx->ecx != MSR_K6_EFER)
12130 {
12131 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12132 pMixedCtx->ecx));
12133 HMVMX_RETURN_UNEXPECTED_EXIT();
12134 }
12135 }
12136
12137 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12138 {
12139 VMXMSREXITREAD enmRead;
12140 VMXMSREXITWRITE enmWrite;
12141 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12142 AssertRCReturn(rc2, rc2);
12143 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12144 {
12145 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12146 HMVMX_RETURN_UNEXPECTED_EXIT();
12147 }
12148 }
12149 break;
12150 }
12151 }
12152 }
12153#endif /* VBOX_STRICT */
12154 }
12155 return rc;
12156}
12157
12158
12159/**
12160 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12161 */
12162HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12163{
12164 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12165
12166 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12167 return VINF_EM_RAW_INTERRUPT;
12168}
12169
12170
12171/**
12172 * VM-exit handler for when the TPR value is lowered below the specified
12173 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12174 */
12175HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12176{
12177 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12178 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12179
12180 /*
12181 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12182 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12183 * resume guest execution.
12184 */
12185 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12187 return VINF_SUCCESS;
12188}
12189
12190
12191/**
12192 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12193 * VM-exit.
12194 *
12195 * @retval VINF_SUCCESS when guest execution can continue.
12196 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12197 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12198 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12199 * interpreter.
12200 */
12201HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12202{
12203 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12204 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12205 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12206 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12207 AssertRCReturn(rc, rc);
12208
12209 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12210 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12211 PVM pVM = pVCpu->CTX_SUFF(pVM);
12212 VBOXSTRICTRC rcStrict;
12213 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12214 switch (uAccessType)
12215 {
12216 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12217 {
12218 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12219 AssertRCReturn(rc, rc);
12220
12221 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12222 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12223 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12224 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12225 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12226 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12227 {
12228 case 0: /* CR0 */
12229 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12230 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12231 break;
12232 case 2: /* CR2 */
12233 /* Nothing to do here, CR2 it's not part of the VMCS. */
12234 break;
12235 case 3: /* CR3 */
12236 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12237 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12238 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12239 break;
12240 case 4: /* CR4 */
12241 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12242 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12243 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12244 break;
12245 case 8: /* CR8 */
12246 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12247 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12248 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12249 break;
12250 default:
12251 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12252 break;
12253 }
12254
12255 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12256 break;
12257 }
12258
12259 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12260 {
12261 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12262 AssertRCReturn(rc, rc);
12263
12264 Assert( !pVM->hm.s.fNestedPaging
12265 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12266 || pVCpu->hm.s.fUsingDebugLoop
12267 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12268
12269 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12270 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12271 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12272
12273 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12274 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12275 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12276 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12277 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12278 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12279 VBOXSTRICTRC_VAL(rcStrict)));
12280 break;
12281 }
12282
12283 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12284 {
12285 AssertRCReturn(rc, rc);
12286 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12287 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12288 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12289 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12290 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12291 break;
12292 }
12293
12294 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12295 {
12296 AssertRCReturn(rc, rc);
12297 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12298 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12299 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12300 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12301 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12302 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12303 break;
12304 }
12305
12306 default:
12307 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12308 VERR_VMX_UNEXPECTED_EXCEPTION);
12309 }
12310
12311 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12312 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12313 NOREF(pVM);
12314 return rcStrict;
12315}
12316
12317
12318/**
12319 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12320 * VM-exit.
12321 */
12322HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12323{
12324 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12325 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12326
12327 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12328 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12329 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12330 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12331 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12332 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12333 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12334 AssertRCReturn(rc2, rc2);
12335
12336 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12337 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12338 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12339 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12340 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12341 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12342 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12343 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12344 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12345
12346 /* I/O operation lookup arrays. */
12347 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12348 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12349
12350 VBOXSTRICTRC rcStrict;
12351 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12352 uint32_t const cbInstr = pVmxTransient->cbInstr;
12353 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12354 PVM pVM = pVCpu->CTX_SUFF(pVM);
12355 if (fIOString)
12356 {
12357#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12358 See @bugref{5752#c158}. Should work now. */
12359 /*
12360 * INS/OUTS - I/O String instruction.
12361 *
12362 * Use instruction-information if available, otherwise fall back on
12363 * interpreting the instruction.
12364 */
12365 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12366 fIOWrite ? 'w' : 'r'));
12367 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12368 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12369 {
12370 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12371 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12372 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12373 AssertRCReturn(rc2, rc2);
12374 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12375 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12376 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12377 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12378 if (fIOWrite)
12379 {
12380 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12381 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12382 }
12383 else
12384 {
12385 /*
12386 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12387 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12388 * See Intel Instruction spec. for "INS".
12389 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12390 */
12391 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12392 }
12393 }
12394 else
12395 {
12396 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12397 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12398 AssertRCReturn(rc2, rc2);
12399 rcStrict = IEMExecOne(pVCpu);
12400 }
12401 /** @todo IEM needs to be setting these flags somehow. */
12402 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12403 fUpdateRipAlready = true;
12404#else
12405 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12406 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12407 if (RT_SUCCESS(rcStrict))
12408 {
12409 if (fIOWrite)
12410 {
12411 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12412 (DISCPUMODE)pDis->uAddrMode, cbValue);
12413 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12414 }
12415 else
12416 {
12417 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12418 (DISCPUMODE)pDis->uAddrMode, cbValue);
12419 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12420 }
12421 }
12422 else
12423 {
12424 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12425 pMixedCtx->rip));
12426 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12427 }
12428#endif
12429 }
12430 else
12431 {
12432 /*
12433 * IN/OUT - I/O instruction.
12434 */
12435 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12436 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12437 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12438 if (fIOWrite)
12439 {
12440 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12441 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12442 }
12443 else
12444 {
12445 uint32_t u32Result = 0;
12446 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12447 if (IOM_SUCCESS(rcStrict))
12448 {
12449 /* Save result of I/O IN instr. in AL/AX/EAX. */
12450 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12451 }
12452 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12453 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12454 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12455 }
12456 }
12457
12458 if (IOM_SUCCESS(rcStrict))
12459 {
12460 if (!fUpdateRipAlready)
12461 {
12462 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12463 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12464 }
12465
12466 /*
12467 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12468 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12469 */
12470 if (fIOString)
12471 {
12472 /** @todo Single-step for INS/OUTS with REP prefix? */
12473 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12474 }
12475 else if ( !fDbgStepping
12476 && fGstStepping)
12477 {
12478 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12479 }
12480
12481 /*
12482 * If any I/O breakpoints are armed, we need to check if one triggered
12483 * and take appropriate action.
12484 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12485 */
12486 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12487 AssertRCReturn(rc2, rc2);
12488
12489 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12490 * execution engines about whether hyper BPs and such are pending. */
12491 uint32_t const uDr7 = pMixedCtx->dr[7];
12492 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12493 && X86_DR7_ANY_RW_IO(uDr7)
12494 && (pMixedCtx->cr4 & X86_CR4_DE))
12495 || DBGFBpIsHwIoArmed(pVM)))
12496 {
12497 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12498
12499 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12500 VMMRZCallRing3Disable(pVCpu);
12501 HM_DISABLE_PREEMPT();
12502
12503 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12504
12505 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12506 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12507 {
12508 /* Raise #DB. */
12509 if (fIsGuestDbgActive)
12510 ASMSetDR6(pMixedCtx->dr[6]);
12511 if (pMixedCtx->dr[7] != uDr7)
12512 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12513
12514 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12515 }
12516 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12517 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12518 else if ( rcStrict2 != VINF_SUCCESS
12519 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12520 rcStrict = rcStrict2;
12521 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12522
12523 HM_RESTORE_PREEMPT();
12524 VMMRZCallRing3Enable(pVCpu);
12525 }
12526 }
12527
12528#ifdef VBOX_STRICT
12529 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12530 Assert(!fIOWrite);
12531 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12532 Assert(fIOWrite);
12533 else
12534 {
12535#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12536 * statuses, that the VMM device and some others may return. See
12537 * IOM_SUCCESS() for guidance. */
12538 AssertMsg( RT_FAILURE(rcStrict)
12539 || rcStrict == VINF_SUCCESS
12540 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12541 || rcStrict == VINF_EM_DBG_BREAKPOINT
12542 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12543 || rcStrict == VINF_EM_RAW_TO_R3
12544 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12545#endif
12546 }
12547#endif
12548
12549 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12550 return rcStrict;
12551}
12552
12553
12554/**
12555 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12556 * VM-exit.
12557 */
12558HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12559{
12560 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12561
12562 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12563 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12564 AssertRCReturn(rc, rc);
12565 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12566 {
12567 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12568 AssertRCReturn(rc, rc);
12569 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12570 {
12571 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12572
12573 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12574 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12575
12576 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12577 Assert(!pVCpu->hm.s.Event.fPending);
12578 pVCpu->hm.s.Event.fPending = true;
12579 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12580 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12581 AssertRCReturn(rc, rc);
12582 if (fErrorCodeValid)
12583 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12584 else
12585 pVCpu->hm.s.Event.u32ErrCode = 0;
12586 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12587 && uVector == X86_XCPT_PF)
12588 {
12589 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12590 }
12591
12592 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12593 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12594 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12595 }
12596 }
12597
12598 /* Fall back to the interpreter to emulate the task-switch. */
12599 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12600 return VERR_EM_INTERPRETER;
12601}
12602
12603
12604/**
12605 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12606 */
12607HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12608{
12609 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12610 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12611 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12612 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12613 AssertRCReturn(rc, rc);
12614 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12615 return VINF_EM_DBG_STEPPED;
12616}
12617
12618
12619/**
12620 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12621 */
12622HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12623{
12624 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12625
12626 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12627
12628 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12629 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12630 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12631 {
12632 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12633 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12634 {
12635 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12636 return VERR_EM_INTERPRETER;
12637 }
12638 }
12639 else
12640 {
12641 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12642 rcStrict1 = VINF_SUCCESS;
12643 return rcStrict1;
12644 }
12645
12646#if 0
12647 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12648 * just sync the whole thing. */
12649 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12650#else
12651 /* Aggressive state sync. for now. */
12652 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12653 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12654 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12655#endif
12656 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12657 AssertRCReturn(rc, rc);
12658
12659 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12660 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12661 VBOXSTRICTRC rcStrict2;
12662 switch (uAccessType)
12663 {
12664 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12665 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12666 {
12667 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12668 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12669 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12670
12671 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12672 GCPhys &= PAGE_BASE_GC_MASK;
12673 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12674 PVM pVM = pVCpu->CTX_SUFF(pVM);
12675 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12676 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12677
12678 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12679 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12680 CPUMCTX2CORE(pMixedCtx), GCPhys);
12681 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12682 if ( rcStrict2 == VINF_SUCCESS
12683 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12684 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12685 {
12686 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12687 | HM_CHANGED_GUEST_RSP
12688 | HM_CHANGED_GUEST_RFLAGS
12689 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12690 rcStrict2 = VINF_SUCCESS;
12691 }
12692 break;
12693 }
12694
12695 default:
12696 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12697 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12698 break;
12699 }
12700
12701 if (rcStrict2 != VINF_SUCCESS)
12702 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12703 return rcStrict2;
12704}
12705
12706
12707/**
12708 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12709 * VM-exit.
12710 */
12711HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12712{
12713 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12714
12715 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12716 if (pVmxTransient->fWasGuestDebugStateActive)
12717 {
12718 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12719 HMVMX_RETURN_UNEXPECTED_EXIT();
12720 }
12721
12722 if ( !pVCpu->hm.s.fSingleInstruction
12723 && !pVmxTransient->fWasHyperDebugStateActive)
12724 {
12725 Assert(!DBGFIsStepping(pVCpu));
12726 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12727
12728 /* Don't intercept MOV DRx any more. */
12729 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12730 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12731 AssertRCReturn(rc, rc);
12732
12733 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12734 VMMRZCallRing3Disable(pVCpu);
12735 HM_DISABLE_PREEMPT();
12736
12737 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12738 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12739 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12740
12741 HM_RESTORE_PREEMPT();
12742 VMMRZCallRing3Enable(pVCpu);
12743
12744#ifdef VBOX_WITH_STATISTICS
12745 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12746 AssertRCReturn(rc, rc);
12747 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12749 else
12750 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12751#endif
12752 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12753 return VINF_SUCCESS;
12754 }
12755
12756 /*
12757 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12758 * Update the segment registers and DR7 from the CPU.
12759 */
12760 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12761 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12762 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12763 AssertRCReturn(rc, rc);
12764 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12765
12766 PVM pVM = pVCpu->CTX_SUFF(pVM);
12767 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12768 {
12769 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12770 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12771 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12772 if (RT_SUCCESS(rc))
12773 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12774 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12775 }
12776 else
12777 {
12778 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12779 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12780 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12781 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12782 }
12783
12784 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12785 if (RT_SUCCESS(rc))
12786 {
12787 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12788 AssertRCReturn(rc2, rc2);
12789 return VINF_SUCCESS;
12790 }
12791 return rc;
12792}
12793
12794
12795/**
12796 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12797 * Conditional VM-exit.
12798 */
12799HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12800{
12801 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12802 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12803
12804 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12805 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12806 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12807 {
12808 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12809 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12810 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12811 {
12812 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12813 return VERR_EM_INTERPRETER;
12814 }
12815 }
12816 else
12817 {
12818 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12819 rcStrict1 = VINF_SUCCESS;
12820 return rcStrict1;
12821 }
12822
12823 RTGCPHYS GCPhys = 0;
12824 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12825
12826#if 0
12827 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12828#else
12829 /* Aggressive state sync. for now. */
12830 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12831 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12832 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12833#endif
12834 AssertRCReturn(rc, rc);
12835
12836 /*
12837 * If we succeed, resume guest execution.
12838 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12839 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12840 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12841 * weird case. See @bugref{6043}.
12842 */
12843 PVM pVM = pVCpu->CTX_SUFF(pVM);
12844 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12845 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12846 if ( rcStrict2 == VINF_SUCCESS
12847 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12848 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12849 {
12850 /* Successfully handled MMIO operation. */
12851 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12852 | HM_CHANGED_GUEST_RSP
12853 | HM_CHANGED_GUEST_RFLAGS
12854 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12855 return VINF_SUCCESS;
12856 }
12857 return rcStrict2;
12858}
12859
12860
12861/**
12862 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12863 * VM-exit.
12864 */
12865HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12866{
12867 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12868 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12869
12870 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12871 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12872 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12873 {
12874 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12875 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12876 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12877 }
12878 else
12879 {
12880 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12881 rcStrict1 = VINF_SUCCESS;
12882 return rcStrict1;
12883 }
12884
12885 RTGCPHYS GCPhys = 0;
12886 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12887 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12888#if 0
12889 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12890#else
12891 /* Aggressive state sync. for now. */
12892 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12893 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12894 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12895#endif
12896 AssertRCReturn(rc, rc);
12897
12898 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12899 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12900
12901 RTGCUINT uErrorCode = 0;
12902 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12903 uErrorCode |= X86_TRAP_PF_ID;
12904 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12905 uErrorCode |= X86_TRAP_PF_RW;
12906 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12907 uErrorCode |= X86_TRAP_PF_P;
12908
12909 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12910
12911 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12912 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12913
12914 /* Handle the pagefault trap for the nested shadow table. */
12915 PVM pVM = pVCpu->CTX_SUFF(pVM);
12916 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12917 TRPMResetTrap(pVCpu);
12918
12919 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12920 if ( rcStrict2 == VINF_SUCCESS
12921 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12922 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12923 {
12924 /* Successfully synced our nested page tables. */
12925 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12926 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12927 | HM_CHANGED_GUEST_RSP
12928 | HM_CHANGED_GUEST_RFLAGS);
12929 return VINF_SUCCESS;
12930 }
12931
12932 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12933 return rcStrict2;
12934}
12935
12936/** @} */
12937
12938/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12939/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12940/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12941
12942/** @name VM-exit exception handlers.
12943 * @{
12944 */
12945
12946/**
12947 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12948 */
12949static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12950{
12951 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12953
12954 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12955 AssertRCReturn(rc, rc);
12956
12957 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12958 {
12959 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12960 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12961
12962 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12963 * provides VM-exit instruction length. If this causes problem later,
12964 * disassemble the instruction like it's done on AMD-V. */
12965 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12966 AssertRCReturn(rc2, rc2);
12967 return rc;
12968 }
12969
12970 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12971 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12972 return rc;
12973}
12974
12975
12976/**
12977 * VM-exit exception handler for \#BP (Breakpoint exception).
12978 */
12979static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12980{
12981 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12983
12984 /** @todo Try optimize this by not saving the entire guest state unless
12985 * really needed. */
12986 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12987 AssertRCReturn(rc, rc);
12988
12989 PVM pVM = pVCpu->CTX_SUFF(pVM);
12990 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12991 if (rc == VINF_EM_RAW_GUEST_TRAP)
12992 {
12993 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12994 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12995 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12996 AssertRCReturn(rc, rc);
12997
12998 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12999 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13000 }
13001
13002 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13003 return rc;
13004}
13005
13006
13007/**
13008 * VM-exit exception handler for \#AC (alignment check exception).
13009 */
13010static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13011{
13012 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13013
13014 /*
13015 * Re-inject it. We'll detect any nesting before getting here.
13016 */
13017 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13018 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13019 AssertRCReturn(rc, rc);
13020 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13021
13022 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13023 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13024 return VINF_SUCCESS;
13025}
13026
13027
13028/**
13029 * VM-exit exception handler for \#DB (Debug exception).
13030 */
13031static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13032{
13033 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13034 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13035 Log6(("XcptDB\n"));
13036
13037 /*
13038 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13039 * for processing.
13040 */
13041 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13042 AssertRCReturn(rc, rc);
13043
13044 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13045 uint64_t uDR6 = X86_DR6_INIT_VAL;
13046 uDR6 |= ( pVmxTransient->uExitQualification
13047 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13048
13049 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13050 if (rc == VINF_EM_RAW_GUEST_TRAP)
13051 {
13052 /*
13053 * The exception was for the guest. Update DR6, DR7.GD and
13054 * IA32_DEBUGCTL.LBR before forwarding it.
13055 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13056 */
13057 VMMRZCallRing3Disable(pVCpu);
13058 HM_DISABLE_PREEMPT();
13059
13060 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13061 pMixedCtx->dr[6] |= uDR6;
13062 if (CPUMIsGuestDebugStateActive(pVCpu))
13063 ASMSetDR6(pMixedCtx->dr[6]);
13064
13065 HM_RESTORE_PREEMPT();
13066 VMMRZCallRing3Enable(pVCpu);
13067
13068 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13069 AssertRCReturn(rc, rc);
13070
13071 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13072 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13073
13074 /* Paranoia. */
13075 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13076 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13077
13078 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13079 AssertRCReturn(rc, rc);
13080
13081 /*
13082 * Raise #DB in the guest.
13083 *
13084 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13085 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13086 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13087 *
13088 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13089 */
13090 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13091 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13092 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13093 AssertRCReturn(rc, rc);
13094 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13095 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13096 return VINF_SUCCESS;
13097 }
13098
13099 /*
13100 * Not a guest trap, must be a hypervisor related debug event then.
13101 * Update DR6 in case someone is interested in it.
13102 */
13103 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13104 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13105 CPUMSetHyperDR6(pVCpu, uDR6);
13106
13107 return rc;
13108}
13109
13110
13111/**
13112 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13113 * point exception).
13114 */
13115static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13116{
13117 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13118
13119 /* We require CR0 and EFER. EFER is always up-to-date. */
13120 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13121 AssertRCReturn(rc, rc);
13122
13123 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13124 VMMRZCallRing3Disable(pVCpu);
13125 HM_DISABLE_PREEMPT();
13126
13127 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
13128 if (pVmxTransient->fWasGuestFPUStateActive)
13129 {
13130 rc = VINF_EM_RAW_GUEST_TRAP;
13131 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13132 }
13133 else
13134 {
13135#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13136 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13137#endif
13138 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13139 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13140 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13141 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13142 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13143 }
13144
13145 HM_RESTORE_PREEMPT();
13146 VMMRZCallRing3Enable(pVCpu);
13147
13148 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13149 {
13150 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13151 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13152 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13153 pVCpu->hm.s.fPreloadGuestFpu = true;
13154 }
13155 else
13156 {
13157 /* Forward #NM to the guest. */
13158 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13159 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13160 AssertRCReturn(rc, rc);
13161 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13162 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13163 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13164 }
13165
13166 return VINF_SUCCESS;
13167}
13168
13169
13170/**
13171 * VM-exit exception handler for \#GP (General-protection exception).
13172 *
13173 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13174 */
13175static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13176{
13177 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13178 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13179
13180 int rc;
13181 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13182 { /* likely */ }
13183 else
13184 {
13185#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13186 Assert(pVCpu->hm.s.fUsingDebugLoop);
13187#endif
13188 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13189 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13190 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13191 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13192 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13193 AssertRCReturn(rc, rc);
13194 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13195 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13196 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13197 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13198 return rc;
13199 }
13200
13201 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13202 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13203
13204 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13205 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13206 AssertRCReturn(rc, rc);
13207
13208 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13209 uint32_t cbOp = 0;
13210 PVM pVM = pVCpu->CTX_SUFF(pVM);
13211 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13212 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13213 if (RT_SUCCESS(rc))
13214 {
13215 rc = VINF_SUCCESS;
13216 Assert(cbOp == pDis->cbInstr);
13217 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13218 switch (pDis->pCurInstr->uOpcode)
13219 {
13220 case OP_CLI:
13221 {
13222 pMixedCtx->eflags.Bits.u1IF = 0;
13223 pMixedCtx->eflags.Bits.u1RF = 0;
13224 pMixedCtx->rip += pDis->cbInstr;
13225 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13226 if ( !fDbgStepping
13227 && pMixedCtx->eflags.Bits.u1TF)
13228 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13229 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13230 break;
13231 }
13232
13233 case OP_STI:
13234 {
13235 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13236 pMixedCtx->eflags.Bits.u1IF = 1;
13237 pMixedCtx->eflags.Bits.u1RF = 0;
13238 pMixedCtx->rip += pDis->cbInstr;
13239 if (!fOldIF)
13240 {
13241 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13242 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13243 }
13244 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13245 if ( !fDbgStepping
13246 && pMixedCtx->eflags.Bits.u1TF)
13247 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13249 break;
13250 }
13251
13252 case OP_HLT:
13253 {
13254 rc = VINF_EM_HALT;
13255 pMixedCtx->rip += pDis->cbInstr;
13256 pMixedCtx->eflags.Bits.u1RF = 0;
13257 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13258 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13259 break;
13260 }
13261
13262 case OP_POPF:
13263 {
13264 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13265 uint32_t cbParm;
13266 uint32_t uMask;
13267 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13268 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13269 {
13270 cbParm = 4;
13271 uMask = 0xffffffff;
13272 }
13273 else
13274 {
13275 cbParm = 2;
13276 uMask = 0xffff;
13277 }
13278
13279 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13280 RTGCPTR GCPtrStack = 0;
13281 X86EFLAGS Eflags;
13282 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13283 &GCPtrStack);
13284 if (RT_SUCCESS(rc))
13285 {
13286 Assert(sizeof(Eflags.u32) >= cbParm);
13287 Eflags.u32 = 0;
13288 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13289 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13290 }
13291 if (RT_FAILURE(rc))
13292 {
13293 rc = VERR_EM_INTERPRETER;
13294 break;
13295 }
13296 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13297 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13298 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13299 pMixedCtx->esp += cbParm;
13300 pMixedCtx->esp &= uMask;
13301 pMixedCtx->rip += pDis->cbInstr;
13302 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13303 | HM_CHANGED_GUEST_RSP
13304 | HM_CHANGED_GUEST_RFLAGS);
13305 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13306 POPF restores EFLAGS.TF. */
13307 if ( !fDbgStepping
13308 && fGstStepping)
13309 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13310 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13311 break;
13312 }
13313
13314 case OP_PUSHF:
13315 {
13316 uint32_t cbParm;
13317 uint32_t uMask;
13318 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13319 {
13320 cbParm = 4;
13321 uMask = 0xffffffff;
13322 }
13323 else
13324 {
13325 cbParm = 2;
13326 uMask = 0xffff;
13327 }
13328
13329 /* Get the stack pointer & push the contents of eflags onto the stack. */
13330 RTGCPTR GCPtrStack = 0;
13331 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13332 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13333 if (RT_FAILURE(rc))
13334 {
13335 rc = VERR_EM_INTERPRETER;
13336 break;
13337 }
13338 X86EFLAGS Eflags = pMixedCtx->eflags;
13339 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13340 Eflags.Bits.u1RF = 0;
13341 Eflags.Bits.u1VM = 0;
13342
13343 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13344 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13345 {
13346 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13347 rc = VERR_EM_INTERPRETER;
13348 break;
13349 }
13350 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13351 pMixedCtx->esp -= cbParm;
13352 pMixedCtx->esp &= uMask;
13353 pMixedCtx->rip += pDis->cbInstr;
13354 pMixedCtx->eflags.Bits.u1RF = 0;
13355 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13356 | HM_CHANGED_GUEST_RSP
13357 | HM_CHANGED_GUEST_RFLAGS);
13358 if ( !fDbgStepping
13359 && pMixedCtx->eflags.Bits.u1TF)
13360 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13361 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13362 break;
13363 }
13364
13365 case OP_IRET:
13366 {
13367 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13368 * instruction reference. */
13369 RTGCPTR GCPtrStack = 0;
13370 uint32_t uMask = 0xffff;
13371 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13372 uint16_t aIretFrame[3];
13373 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13374 {
13375 rc = VERR_EM_INTERPRETER;
13376 break;
13377 }
13378 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13379 &GCPtrStack);
13380 if (RT_SUCCESS(rc))
13381 {
13382 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13383 PGMACCESSORIGIN_HM));
13384 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13385 }
13386 if (RT_FAILURE(rc))
13387 {
13388 rc = VERR_EM_INTERPRETER;
13389 break;
13390 }
13391 pMixedCtx->eip = 0;
13392 pMixedCtx->ip = aIretFrame[0];
13393 pMixedCtx->cs.Sel = aIretFrame[1];
13394 pMixedCtx->cs.ValidSel = aIretFrame[1];
13395 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13396 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13397 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13398 pMixedCtx->sp += sizeof(aIretFrame);
13399 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13400 | HM_CHANGED_GUEST_SEGMENT_REGS
13401 | HM_CHANGED_GUEST_RSP
13402 | HM_CHANGED_GUEST_RFLAGS);
13403 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13404 if ( !fDbgStepping
13405 && fGstStepping)
13406 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13407 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13408 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13409 break;
13410 }
13411
13412 case OP_INT:
13413 {
13414 uint16_t uVector = pDis->Param1.uValue & 0xff;
13415 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13416 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13417 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13418 break;
13419 }
13420
13421 case OP_INTO:
13422 {
13423 if (pMixedCtx->eflags.Bits.u1OF)
13424 {
13425 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13426 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13427 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13428 }
13429 else
13430 {
13431 pMixedCtx->eflags.Bits.u1RF = 0;
13432 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13433 }
13434 break;
13435 }
13436
13437 default:
13438 {
13439 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13440 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13441 EMCODETYPE_SUPERVISOR);
13442 rc = VBOXSTRICTRC_VAL(rc2);
13443 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13444 /** @todo We have to set pending-debug exceptions here when the guest is
13445 * single-stepping depending on the instruction that was interpreted. */
13446 Log4(("#GP rc=%Rrc\n", rc));
13447 break;
13448 }
13449 }
13450 }
13451 else
13452 rc = VERR_EM_INTERPRETER;
13453
13454 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13455 ("#GP Unexpected rc=%Rrc\n", rc));
13456 return rc;
13457}
13458
13459
13460/**
13461 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13462 * the exception reported in the VMX transient structure back into the VM.
13463 *
13464 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13465 * up-to-date.
13466 */
13467static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13468{
13469 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13470#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13471 Assert(pVCpu->hm.s.fUsingDebugLoop);
13472#endif
13473
13474 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13475 hmR0VmxCheckExitDueToEventDelivery(). */
13476 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13477 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13478 AssertRCReturn(rc, rc);
13479 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13480
13481#ifdef DEBUG_ramshankar
13482 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13483 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13484 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13485#endif
13486
13487 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13488 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13489 return VINF_SUCCESS;
13490}
13491
13492
13493/**
13494 * VM-exit exception handler for \#PF (Page-fault exception).
13495 */
13496static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13497{
13498 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13499 PVM pVM = pVCpu->CTX_SUFF(pVM);
13500 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13501 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13502 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13503 AssertRCReturn(rc, rc);
13504
13505 if (!pVM->hm.s.fNestedPaging)
13506 { /* likely */ }
13507 else
13508 {
13509#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13510 Assert(pVCpu->hm.s.fUsingDebugLoop);
13511#endif
13512 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13513 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13514 {
13515 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13516 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13517 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13518 }
13519 else
13520 {
13521 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13522 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13523 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13524 }
13525 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13526 return rc;
13527 }
13528
13529 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13530 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13531 if (pVmxTransient->fVectoringPF)
13532 {
13533 Assert(pVCpu->hm.s.Event.fPending);
13534 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13535 }
13536
13537 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13538 AssertRCReturn(rc, rc);
13539
13540 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13541 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13542
13543 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13544 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13545 (RTGCPTR)pVmxTransient->uExitQualification);
13546
13547 Log4(("#PF: rc=%Rrc\n", rc));
13548 if (rc == VINF_SUCCESS)
13549 {
13550#if 0
13551 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13552 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13553 * memory? We don't update the whole state here... */
13554 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13555 | HM_CHANGED_GUEST_RSP
13556 | HM_CHANGED_GUEST_RFLAGS
13557 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13558#else
13559 /*
13560 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13561 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13562 */
13563 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13564 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13565#endif
13566 TRPMResetTrap(pVCpu);
13567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13568 return rc;
13569 }
13570
13571 if (rc == VINF_EM_RAW_GUEST_TRAP)
13572 {
13573 if (!pVmxTransient->fVectoringDoublePF)
13574 {
13575 /* It's a guest page fault and needs to be reflected to the guest. */
13576 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13577 TRPMResetTrap(pVCpu);
13578 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13579 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13580 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13581 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13582 }
13583 else
13584 {
13585 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13586 TRPMResetTrap(pVCpu);
13587 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13588 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13589 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13590 }
13591
13592 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13593 return VINF_SUCCESS;
13594 }
13595
13596 TRPMResetTrap(pVCpu);
13597 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13598 return rc;
13599}
13600
13601/** @} */
13602
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette