VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp@ 98103

Last change on this file since 98103 was 98103, checked in by vboxsync, 17 months ago

Copyright year updates by scm.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 69.5 KB
Line 
1/* $Id: HMR0.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
2/** @file
3 * Hardware Assisted Virtualization Manager (HM) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_HM
33#define VMCPU_INCL_CPUM_GST_CTX
34#include <VBox/vmm/hm.h>
35#include <VBox/vmm/pgm.h>
36#include "HMInternal.h"
37#include <VBox/vmm/vmcc.h>
38#include <VBox/vmm/hm_svm.h>
39#include <VBox/vmm/hmvmxinline.h>
40#include <VBox/err.h>
41#include <VBox/log.h>
42#include <iprt/assert.h>
43#include <iprt/asm.h>
44#include <iprt/asm-amd64-x86.h>
45#include <iprt/cpuset.h>
46#include <iprt/mem.h>
47#include <iprt/memobj.h>
48#include <iprt/once.h>
49#include <iprt/param.h>
50#include <iprt/power.h>
51#include <iprt/string.h>
52#include <iprt/thread.h>
53#include <iprt/x86.h>
54#include "HMVMXR0.h"
55#include "HMSVMR0.h"
56
57
58/*********************************************************************************************************************************
59* Internal Functions *
60*********************************************************************************************************************************/
61static DECLCALLBACK(void) hmR0EnableCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2);
62static DECLCALLBACK(void) hmR0DisableCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2);
63static DECLCALLBACK(void) hmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser);
64static DECLCALLBACK(void) hmR0MpEventCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData);
65
66
67/*********************************************************************************************************************************
68* Structures and Typedefs *
69*********************************************************************************************************************************/
70/**
71 * This is used to manage the status code of a RTMpOnAll in HM.
72 */
73typedef struct HMR0FIRSTRC
74{
75 /** The status code. */
76 int32_t volatile rc;
77 /** The ID of the CPU reporting the first failure. */
78 RTCPUID volatile idCpu;
79} HMR0FIRSTRC;
80/** Pointer to a first return code structure. */
81typedef HMR0FIRSTRC *PHMR0FIRSTRC;
82
83/**
84 * Ring-0 method table for AMD-V and VT-x specific operations.
85 */
86typedef struct HMR0VTABLE
87{
88 DECLR0CALLBACKMEMBER(int, pfnEnterSession, (PVMCPUCC pVCpu));
89 DECLR0CALLBACKMEMBER(void, pfnThreadCtxCallback, (RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit));
90 DECLR0CALLBACKMEMBER(int, pfnAssertionCallback, (PVMCPUCC pVCpu));
91 DECLR0CALLBACKMEMBER(int, pfnExportHostState, (PVMCPUCC pVCpu));
92 DECLR0CALLBACKMEMBER(VBOXSTRICTRC, pfnRunGuestCode, (PVMCPUCC pVCpu));
93 DECLR0CALLBACKMEMBER(int, pfnEnableCpu, (PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage,
94 bool fEnabledByHost, PCSUPHWVIRTMSRS pHwvirtMsrs));
95 DECLR0CALLBACKMEMBER(int, pfnDisableCpu, (PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage));
96 DECLR0CALLBACKMEMBER(int, pfnInitVM, (PVMCC pVM));
97 DECLR0CALLBACKMEMBER(int, pfnTermVM, (PVMCC pVM));
98 DECLR0CALLBACKMEMBER(int, pfnSetupVM, (PVMCC pVM));
99} HMR0VTABLE;
100
101
102/*********************************************************************************************************************************
103* Global Variables *
104*********************************************************************************************************************************/
105/** The active ring-0 HM operations (copied from one of the table at init). */
106static HMR0VTABLE g_HmR0Ops;
107/** Indicates whether the host is suspending or not. We'll refuse a few
108 * actions when the host is being suspended to speed up the suspending and
109 * avoid trouble. */
110static bool volatile g_fHmSuspended;
111/** If set, VT-x/AMD-V is enabled globally at init time, otherwise it's
112 * enabled and disabled each time it's used to execute guest code. */
113static bool g_fHmGlobalInit;
114/** Host kernel flags that HM might need to know (SUPKERNELFEATURES_XXX). */
115uint32_t g_fHmHostKernelFeatures;
116/** Maximum allowed ASID/VPID (inclusive).
117 * @todo r=bird: This is exclusive for VT-x according to source code comment.
118 * Couldn't immediately find any docs on AMD-V, but suspect it is
119 * exclusive there as well given how hmR0SvmFlushTaggedTlb() use it. */
120uint32_t g_uHmMaxAsid;
121
122
123/** Set if VT-x (VMX) is supported by the CPU. */
124bool g_fHmVmxSupported = false;
125/** VMX: Whether we're using the preemption timer or not. */
126bool g_fHmVmxUsePreemptTimer;
127/** VMX: The shift mask employed by the VMX-Preemption timer. */
128uint8_t g_cHmVmxPreemptTimerShift;
129/** VMX: Set if swapping EFER is supported. */
130bool g_fHmVmxSupportsVmcsEfer = false;
131/** VMX: Whether we're using SUPR0EnableVTx or not. */
132static bool g_fHmVmxUsingSUPR0EnableVTx = false;
133/** VMX: Set if we've called SUPR0EnableVTx(true) and should disable it during
134 * module termination. */
135static bool g_fHmVmxCalledSUPR0EnableVTx = false;
136/** VMX: Host CR4 value (set by ring-0 VMX init) */
137uint64_t g_uHmVmxHostCr4;
138/** VMX: Host EFER value (set by ring-0 VMX init) */
139uint64_t g_uHmVmxHostMsrEfer;
140/** VMX: Host SMM monitor control (used for logging/diagnostics) */
141uint64_t g_uHmVmxHostSmmMonitorCtl;
142
143
144/** Set if AMD-V is supported by the CPU. */
145bool g_fHmSvmSupported = false;
146/** SVM revision. */
147uint32_t g_uHmSvmRev;
148/** SVM feature bits from cpuid 0x8000000a */
149uint32_t g_fHmSvmFeatures;
150
151
152/** MSRs. */
153SUPHWVIRTMSRS g_HmMsrs;
154
155/** Last recorded error code during HM ring-0 init. */
156static int32_t g_rcHmInit = VINF_SUCCESS;
157
158/** Per CPU globals. */
159static HMPHYSCPU g_aHmCpuInfo[RTCPUSET_MAX_CPUS];
160
161/** Whether we've already initialized all CPUs.
162 * @remarks We could check the EnableAllCpusOnce state, but this is
163 * simpler and hopefully easier to understand. */
164static bool g_fHmEnabled = false;
165/** Serialize initialization in HMR0EnableAllCpus. */
166static RTONCE g_HmEnableAllCpusOnce = RTONCE_INITIALIZER;
167
168
169/** HM ring-0 operations for VT-x. */
170static HMR0VTABLE const g_HmR0OpsVmx =
171{
172 /* .pfnEnterSession = */ VMXR0Enter,
173 /* .pfnThreadCtxCallback = */ VMXR0ThreadCtxCallback,
174 /* .pfnAssertionCallback = */ VMXR0AssertionCallback,
175 /* .pfnExportHostState = */ VMXR0ExportHostState,
176 /* .pfnRunGuestCode = */ VMXR0RunGuestCode,
177 /* .pfnEnableCpu = */ VMXR0EnableCpu,
178 /* .pfnDisableCpu = */ VMXR0DisableCpu,
179 /* .pfnInitVM = */ VMXR0InitVM,
180 /* .pfnTermVM = */ VMXR0TermVM,
181 /* .pfnSetupVM = */ VMXR0SetupVM,
182};
183
184/** HM ring-0 operations for AMD-V. */
185static HMR0VTABLE const g_HmR0OpsSvm =
186{
187 /* .pfnEnterSession = */ SVMR0Enter,
188 /* .pfnThreadCtxCallback = */ SVMR0ThreadCtxCallback,
189 /* .pfnAssertionCallback = */ SVMR0AssertionCallback,
190 /* .pfnExportHostState = */ SVMR0ExportHostState,
191 /* .pfnRunGuestCode = */ SVMR0RunGuestCode,
192 /* .pfnEnableCpu = */ SVMR0EnableCpu,
193 /* .pfnDisableCpu = */ SVMR0DisableCpu,
194 /* .pfnInitVM = */ SVMR0InitVM,
195 /* .pfnTermVM = */ SVMR0TermVM,
196 /* .pfnSetupVM = */ SVMR0SetupVM,
197};
198
199
200/** @name Dummy callback handlers for when neither VT-x nor AMD-V is supported.
201 * @{ */
202
203static DECLCALLBACK(int) hmR0DummyEnter(PVMCPUCC pVCpu)
204{
205 RT_NOREF(pVCpu);
206 return VINF_SUCCESS;
207}
208
209static DECLCALLBACK(void) hmR0DummyThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
210{
211 RT_NOREF(enmEvent, pVCpu, fGlobalInit);
212}
213
214static DECLCALLBACK(int) hmR0DummyEnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage,
215 bool fEnabledBySystem, PCSUPHWVIRTMSRS pHwvirtMsrs)
216{
217 RT_NOREF(pHostCpu, pVM, pvCpuPage, HCPhysCpuPage, fEnabledBySystem, pHwvirtMsrs);
218 return VINF_SUCCESS;
219}
220
221static DECLCALLBACK(int) hmR0DummyDisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
222{
223 RT_NOREF(pHostCpu, pvCpuPage, HCPhysCpuPage);
224 return VINF_SUCCESS;
225}
226
227static DECLCALLBACK(int) hmR0DummyInitVM(PVMCC pVM)
228{
229 RT_NOREF(pVM);
230 return VINF_SUCCESS;
231}
232
233static DECLCALLBACK(int) hmR0DummyTermVM(PVMCC pVM)
234{
235 RT_NOREF(pVM);
236 return VINF_SUCCESS;
237}
238
239static DECLCALLBACK(int) hmR0DummySetupVM(PVMCC pVM)
240{
241 RT_NOREF(pVM);
242 return VINF_SUCCESS;
243}
244
245static DECLCALLBACK(int) hmR0DummyAssertionCallback(PVMCPUCC pVCpu)
246{
247 RT_NOREF(pVCpu);
248 return VINF_SUCCESS;
249}
250
251static DECLCALLBACK(VBOXSTRICTRC) hmR0DummyRunGuestCode(PVMCPUCC pVCpu)
252{
253 RT_NOREF(pVCpu);
254 return VERR_NOT_SUPPORTED;
255}
256
257static DECLCALLBACK(int) hmR0DummyExportHostState(PVMCPUCC pVCpu)
258{
259 RT_NOREF(pVCpu);
260 return VINF_SUCCESS;
261}
262
263/** Dummy ops. */
264static HMR0VTABLE const g_HmR0OpsDummy =
265{
266 /* .pfnEnterSession = */ hmR0DummyEnter,
267 /* .pfnThreadCtxCallback = */ hmR0DummyThreadCtxCallback,
268 /* .pfnAssertionCallback = */ hmR0DummyAssertionCallback,
269 /* .pfnExportHostState = */ hmR0DummyExportHostState,
270 /* .pfnRunGuestCode = */ hmR0DummyRunGuestCode,
271 /* .pfnEnableCpu = */ hmR0DummyEnableCpu,
272 /* .pfnDisableCpu = */ hmR0DummyDisableCpu,
273 /* .pfnInitVM = */ hmR0DummyInitVM,
274 /* .pfnTermVM = */ hmR0DummyTermVM,
275 /* .pfnSetupVM = */ hmR0DummySetupVM,
276};
277
278/** @} */
279
280
281/**
282 * Initializes a first return code structure.
283 *
284 * @param pFirstRc The structure to init.
285 */
286static void hmR0FirstRcInit(PHMR0FIRSTRC pFirstRc)
287{
288 pFirstRc->rc = VINF_SUCCESS;
289 pFirstRc->idCpu = NIL_RTCPUID;
290}
291
292
293/**
294 * Try set the status code (success ignored).
295 *
296 * @param pFirstRc The first return code structure.
297 * @param rc The status code.
298 */
299static void hmR0FirstRcSetStatus(PHMR0FIRSTRC pFirstRc, int rc)
300{
301 if ( RT_FAILURE(rc)
302 && ASMAtomicCmpXchgS32(&pFirstRc->rc, rc, VINF_SUCCESS))
303 pFirstRc->idCpu = RTMpCpuId();
304}
305
306
307/**
308 * Get the status code of a first return code structure.
309 *
310 * @returns The status code; VINF_SUCCESS or error status, no informational or
311 * warning errors.
312 * @param pFirstRc The first return code structure.
313 */
314static int hmR0FirstRcGetStatus(PHMR0FIRSTRC pFirstRc)
315{
316 return pFirstRc->rc;
317}
318
319
320#ifdef VBOX_STRICT
321# ifndef DEBUG_bird
322/**
323 * Get the CPU ID on which the failure status code was reported.
324 *
325 * @returns The CPU ID, NIL_RTCPUID if no failure was reported.
326 * @param pFirstRc The first return code structure.
327 */
328static RTCPUID hmR0FirstRcGetCpuId(PHMR0FIRSTRC pFirstRc)
329{
330 return pFirstRc->idCpu;
331}
332# endif
333#endif /* VBOX_STRICT */
334
335
336
337/**
338 * Worker function used by hmR0PowerCallback() and HMR0Init() to initalize VT-x
339 * on a CPU.
340 *
341 * @param idCpu The identifier for the CPU the function is called on.
342 * @param pvUser1 Pointer to the first RC structure.
343 * @param pvUser2 Ignored.
344 */
345static DECLCALLBACK(void) hmR0InitIntelCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
346{
347 PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser1;
348 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
349 Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
350 NOREF(idCpu); NOREF(pvUser2);
351
352 int rc = SUPR0GetVmxUsability(NULL /* pfIsSmxModeAmbiguous */);
353 hmR0FirstRcSetStatus(pFirstRc, rc);
354}
355
356
357/**
358 * Intel specific initialization code.
359 *
360 * @returns VBox status code (will only fail if out of memory).
361 */
362static int hmR0InitIntel(void)
363{
364 /* Read this MSR now as it may be useful for error reporting when initializing VT-x fails. */
365 g_HmMsrs.u.vmx.u64FeatCtrl = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
366
367 /*
368 * First try use native kernel API for controlling VT-x.
369 * (This is only supported by some Mac OS X kernels atm.)
370 */
371 int rc;
372 g_rcHmInit = rc = SUPR0EnableVTx(true /* fEnable */);
373 g_fHmVmxUsingSUPR0EnableVTx = rc != VERR_NOT_SUPPORTED;
374 if (g_fHmVmxUsingSUPR0EnableVTx)
375 {
376 AssertLogRelMsg(rc == VINF_SUCCESS || rc == VERR_VMX_IN_VMX_ROOT_MODE || rc == VERR_VMX_NO_VMX, ("%Rrc\n", rc));
377 if (RT_SUCCESS(rc))
378 {
379 g_fHmVmxSupported = true;
380 rc = SUPR0EnableVTx(false /* fEnable */);
381 AssertLogRelRC(rc);
382 rc = VINF_SUCCESS;
383 }
384 }
385 else
386 {
387 HMR0FIRSTRC FirstRc;
388 hmR0FirstRcInit(&FirstRc);
389 g_rcHmInit = rc = RTMpOnAll(hmR0InitIntelCpu, &FirstRc, NULL);
390 if (RT_SUCCESS(rc))
391 g_rcHmInit = rc = hmR0FirstRcGetStatus(&FirstRc);
392 }
393
394 if (RT_SUCCESS(rc))
395 {
396 /* Read CR4 and EFER for logging/diagnostic purposes. */
397 g_uHmVmxHostCr4 = ASMGetCR4();
398 g_uHmVmxHostMsrEfer = ASMRdMsr(MSR_K6_EFER);
399
400 /* Get VMX MSRs (and feature control MSR) for determining VMX features we can ultimately use. */
401 SUPR0GetHwvirtMsrs(&g_HmMsrs, SUPVTCAPS_VT_X, false /* fForce */);
402
403 /*
404 * Nested KVM workaround: Intel SDM section 34.15.5 describes that
405 * MSR_IA32_SMM_MONITOR_CTL depends on bit 49 of MSR_IA32_VMX_BASIC while
406 * table 35-2 says that this MSR is available if either VMX or SMX is supported.
407 */
408 uint64_t const uVmxBasicMsr = g_HmMsrs.u.vmx.u64Basic;
409 if (RT_BF_GET(uVmxBasicMsr, VMX_BF_BASIC_DUAL_MON))
410 g_uHmVmxHostSmmMonitorCtl = ASMRdMsr(MSR_IA32_SMM_MONITOR_CTL);
411
412 /* Initialize VPID - 16 bits ASID. */
413 g_uHmMaxAsid = 0x10000; /* exclusive */
414
415 /*
416 * If the host OS has not enabled VT-x for us, try enter VMX root mode
417 * to really verify if VT-x is usable.
418 */
419 if (!g_fHmVmxUsingSUPR0EnableVTx)
420 {
421 /* Allocate a temporary VMXON region. */
422 RTR0MEMOBJ hScatchMemObj;
423 rc = RTR0MemObjAllocCont(&hScatchMemObj, HOST_PAGE_SIZE, false /* fExecutable */);
424 if (RT_FAILURE(rc))
425 {
426 LogRel(("hmR0InitIntel: RTR0MemObjAllocCont(,HOST_PAGE_SIZE,false) -> %Rrc\n", rc));
427 return rc;
428 }
429 void *pvScatchPage = RTR0MemObjAddress(hScatchMemObj);
430 RTHCPHYS const HCPhysScratchPage = RTR0MemObjGetPagePhysAddr(hScatchMemObj, 0);
431 RT_BZERO(pvScatchPage, HOST_PAGE_SIZE);
432
433 /* Set revision dword at the beginning of the VMXON structure. */
434 *(uint32_t *)pvScatchPage = RT_BF_GET(uVmxBasicMsr, VMX_BF_BASIC_VMCS_ID);
435
436 /* Make sure we don't get rescheduled to another CPU during this probe. */
437 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
438
439 /* Enable CR4.VMXE if it isn't already set. */
440 RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
441
442 /*
443 * The only way of checking if we're in VMX root mode or not is to try and enter it.
444 * There is no instruction or control bit that tells us if we're in VMX root mode.
445 * Therefore, try and enter VMX root mode here.
446 */
447 rc = VMXEnable(HCPhysScratchPage);
448 if (RT_SUCCESS(rc))
449 {
450 g_fHmVmxSupported = true;
451 VMXDisable();
452 }
453 else
454 {
455 /*
456 * KVM leaves the CPU in VMX root mode. Not only is this not allowed,
457 * it will crash the host when we enter raw mode, because:
458 *
459 * (a) clearing X86_CR4_VMXE in CR4 causes a #GP (we no longer modify
460 * this bit), and
461 * (b) turning off paging causes a #GP (unavoidable when switching
462 * from long to 32 bits mode or 32 bits to PAE).
463 *
464 * They should fix their code, but until they do we simply refuse to run.
465 */
466 g_rcHmInit = VERR_VMX_IN_VMX_ROOT_MODE;
467 Assert(g_fHmVmxSupported == false);
468 }
469
470 /* Restore CR4.VMXE if it wasn't set prior to us setting it above. */
471 if (!(uOldCr4 & X86_CR4_VMXE))
472 SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
473
474 /* Restore interrupts. */
475 ASMSetFlags(fEFlags);
476
477 RTR0MemObjFree(hScatchMemObj, false);
478 }
479
480 if (g_fHmVmxSupported)
481 {
482 rc = VMXR0GlobalInit();
483 if (RT_SUCCESS(rc))
484 {
485 /*
486 * Install the VT-x methods.
487 */
488 g_HmR0Ops = g_HmR0OpsVmx;
489
490 /*
491 * Check for the VMX-Preemption Timer and adjust for the "VMX-Preemption
492 * Timer Does Not Count Down at the Rate Specified" CPU erratum.
493 */
494 if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER)
495 {
496 g_fHmVmxUsePreemptTimer = true;
497 g_cHmVmxPreemptTimerShift = RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_PREEMPT_TIMER_TSC);
498 if (HMIsSubjectToVmxPreemptTimerErratum())
499 g_cHmVmxPreemptTimerShift = 0; /* This is about right most of the time here. */
500 }
501 else
502 g_fHmVmxUsePreemptTimer = false;
503
504 /*
505 * Check for EFER swapping support.
506 */
507 g_fHmVmxSupportsVmcsEfer = (g_HmMsrs.u.vmx.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
508 && (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
509 && (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR);
510 }
511 else
512 {
513 g_rcHmInit = rc;
514 g_fHmVmxSupported = false;
515 }
516 }
517 }
518#ifdef LOG_ENABLED
519 else
520 SUPR0Printf("hmR0InitIntelCpu failed with rc=%Rrc\n", g_rcHmInit);
521#endif
522 return VINF_SUCCESS;
523}
524
525
526/**
527 * Worker function used by hmR0PowerCallback() and HMR0Init() to initalize AMD-V
528 * on a CPU.
529 *
530 * @param idCpu The identifier for the CPU the function is called on.
531 * @param pvUser1 Pointer to the first RC structure.
532 * @param pvUser2 Ignored.
533 */
534static DECLCALLBACK(void) hmR0InitAmdCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
535{
536 PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser1;
537 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
538 Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
539 NOREF(idCpu); NOREF(pvUser2);
540
541 int rc = SUPR0GetSvmUsability(true /* fInitSvm */);
542 hmR0FirstRcSetStatus(pFirstRc, rc);
543}
544
545
546/**
547 * AMD-specific initialization code.
548 *
549 * @returns VBox status code (will only fail if out of memory).
550 */
551static int hmR0InitAmd(void)
552{
553 /* Call the global AMD-V initialization routine (should only fail in out-of-memory situations). */
554 int rc = SVMR0GlobalInit();
555 if (RT_SUCCESS(rc))
556 {
557 /*
558 * Install the AMD-V methods.
559 */
560 g_HmR0Ops = g_HmR0OpsSvm;
561
562 /* Query AMD features. */
563 uint32_t u32Dummy;
564 ASMCpuId(0x8000000a, &g_uHmSvmRev, &g_uHmMaxAsid, &u32Dummy, &g_fHmSvmFeatures);
565
566 /*
567 * We need to check if AMD-V has been properly initialized on all CPUs.
568 * Some BIOSes might do a poor job.
569 */
570 HMR0FIRSTRC FirstRc;
571 hmR0FirstRcInit(&FirstRc);
572 rc = RTMpOnAll(hmR0InitAmdCpu, &FirstRc, NULL);
573 AssertRC(rc);
574 if (RT_SUCCESS(rc))
575 rc = hmR0FirstRcGetStatus(&FirstRc);
576#ifndef DEBUG_bird
577 AssertMsg(rc == VINF_SUCCESS || rc == VERR_SVM_IN_USE,
578 ("hmR0InitAmdCpu failed for cpu %d with rc=%Rrc\n", hmR0FirstRcGetCpuId(&FirstRc), rc));
579#endif
580 if (RT_SUCCESS(rc))
581 {
582 SUPR0GetHwvirtMsrs(&g_HmMsrs, SUPVTCAPS_AMD_V, false /* fForce */);
583 g_fHmSvmSupported = true;
584 }
585 else
586 {
587 g_rcHmInit = rc;
588 if (rc == VERR_SVM_DISABLED || rc == VERR_SVM_IN_USE)
589 rc = VINF_SUCCESS; /* Don't fail if AMD-V is disabled or in use. */
590 }
591 }
592 else
593 g_rcHmInit = rc;
594 return rc;
595}
596
597
598/**
599 * Does global Ring-0 HM initialization (at module init).
600 *
601 * @returns VBox status code.
602 */
603VMMR0_INT_DECL(int) HMR0Init(void)
604{
605 /*
606 * Initialize the globals.
607 */
608 g_fHmEnabled = false;
609 for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
610 {
611 g_aHmCpuInfo[i].idCpu = NIL_RTCPUID;
612 g_aHmCpuInfo[i].hMemObj = NIL_RTR0MEMOBJ;
613 g_aHmCpuInfo[i].HCPhysMemObj = NIL_RTHCPHYS;
614 g_aHmCpuInfo[i].pvMemObj = NULL;
615#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
616 g_aHmCpuInfo[i].n.svm.hNstGstMsrpm = NIL_RTR0MEMOBJ;
617 g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm = NIL_RTHCPHYS;
618 g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm = NULL;
619#endif
620 }
621
622 /* Fill in all callbacks with placeholders. */
623 g_HmR0Ops = g_HmR0OpsDummy;
624
625 /* Default is global VT-x/AMD-V init. */
626 g_fHmGlobalInit = true;
627
628 g_fHmVmxSupported = false;
629 g_fHmSvmSupported = false;
630 g_uHmMaxAsid = 0;
631
632 /*
633 * Get host kernel features that HM might need to know in order
634 * to co-operate and function properly with the host OS (e.g. SMAP).
635 */
636 g_fHmHostKernelFeatures = SUPR0GetKernelFeatures();
637
638 /*
639 * Make sure aCpuInfo is big enough for all the CPUs on this system.
640 */
641 if (RTMpGetArraySize() > RT_ELEMENTS(g_aHmCpuInfo))
642 {
643 LogRel(("HM: Too many real CPUs/cores/threads - %u, max %u\n", RTMpGetArraySize(), RT_ELEMENTS(g_aHmCpuInfo)));
644 return VERR_TOO_MANY_CPUS;
645 }
646
647 /*
648 * Check for VT-x or AMD-V support.
649 * Return failure only in out-of-memory situations.
650 */
651 uint32_t fCaps = 0;
652 int rc = SUPR0GetVTSupport(&fCaps);
653 if (RT_SUCCESS(rc))
654 {
655 if (fCaps & SUPVTCAPS_VT_X)
656 rc = hmR0InitIntel();
657 else
658 {
659 Assert(fCaps & SUPVTCAPS_AMD_V);
660 rc = hmR0InitAmd();
661 }
662 if (RT_SUCCESS(rc))
663 {
664 /*
665 * Register notification callbacks that we can use to disable/enable CPUs
666 * when brought offline/online or suspending/resuming.
667 */
668 if (!g_fHmVmxUsingSUPR0EnableVTx)
669 {
670 rc = RTMpNotificationRegister(hmR0MpEventCallback, NULL);
671 if (RT_SUCCESS(rc))
672 {
673 rc = RTPowerNotificationRegister(hmR0PowerCallback, NULL);
674 if (RT_FAILURE(rc))
675 RTMpNotificationDeregister(hmR0MpEventCallback, NULL);
676 }
677 if (RT_FAILURE(rc))
678 {
679 /* There shouldn't be any per-cpu allocations at this point,
680 so just have to call SVMR0GlobalTerm and VMXR0GlobalTerm. */
681 if (fCaps & SUPVTCAPS_VT_X)
682 VMXR0GlobalTerm();
683 else
684 SVMR0GlobalTerm();
685 g_HmR0Ops = g_HmR0OpsDummy;
686 g_rcHmInit = rc;
687 g_fHmSvmSupported = false;
688 g_fHmVmxSupported = false;
689 }
690 }
691 }
692 }
693 else
694 {
695 g_rcHmInit = rc;
696 rc = VINF_SUCCESS; /* We return success here because module init shall not fail if HM fails to initialize. */
697 }
698 return rc;
699}
700
701
702/**
703 * Does global Ring-0 HM termination (at module termination).
704 *
705 * @returns VBox status code (ignored).
706 */
707VMMR0_INT_DECL(int) HMR0Term(void)
708{
709 int rc;
710 if ( g_fHmVmxSupported
711 && g_fHmVmxUsingSUPR0EnableVTx)
712 {
713 /*
714 * Simple if the host OS manages VT-x.
715 */
716 Assert(g_fHmGlobalInit);
717
718 if (g_fHmVmxCalledSUPR0EnableVTx)
719 {
720 rc = SUPR0EnableVTx(false /* fEnable */);
721 g_fHmVmxCalledSUPR0EnableVTx = false;
722 }
723 else
724 rc = VINF_SUCCESS;
725
726 for (unsigned iCpu = 0; iCpu < RT_ELEMENTS(g_aHmCpuInfo); iCpu++)
727 {
728 g_aHmCpuInfo[iCpu].fConfigured = false;
729 Assert(g_aHmCpuInfo[iCpu].hMemObj == NIL_RTR0MEMOBJ);
730 }
731 }
732 else
733 {
734 Assert(!g_fHmVmxSupported || !g_fHmVmxUsingSUPR0EnableVTx);
735
736 /* Doesn't really matter if this fails. */
737 RTMpNotificationDeregister(hmR0MpEventCallback, NULL);
738 RTPowerNotificationDeregister(hmR0PowerCallback, NULL);
739 rc = VINF_SUCCESS;
740
741 /*
742 * Disable VT-x/AMD-V on all CPUs if we enabled it before.
743 */
744 if (g_fHmGlobalInit)
745 {
746 HMR0FIRSTRC FirstRc;
747 hmR0FirstRcInit(&FirstRc);
748 rc = RTMpOnAll(hmR0DisableCpuCallback, NULL /* pvUser 1 */, &FirstRc);
749 Assert(RT_SUCCESS(rc) || rc == VERR_NOT_SUPPORTED);
750 if (RT_SUCCESS(rc))
751 rc = hmR0FirstRcGetStatus(&FirstRc);
752 }
753
754 /*
755 * Free the per-cpu pages used for VT-x and AMD-V.
756 */
757 for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
758 {
759 if (g_aHmCpuInfo[i].hMemObj != NIL_RTR0MEMOBJ)
760 {
761 RTR0MemObjFree(g_aHmCpuInfo[i].hMemObj, false);
762 g_aHmCpuInfo[i].hMemObj = NIL_RTR0MEMOBJ;
763 g_aHmCpuInfo[i].HCPhysMemObj = NIL_RTHCPHYS;
764 g_aHmCpuInfo[i].pvMemObj = NULL;
765 }
766#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
767 if (g_aHmCpuInfo[i].n.svm.hNstGstMsrpm != NIL_RTR0MEMOBJ)
768 {
769 RTR0MemObjFree(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm, false);
770 g_aHmCpuInfo[i].n.svm.hNstGstMsrpm = NIL_RTR0MEMOBJ;
771 g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm = NIL_RTHCPHYS;
772 g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm = NULL;
773 }
774#endif
775 }
776 }
777
778 /** @todo This needs cleaning up. There's no matching
779 * hmR0TermIntel()/hmR0TermAmd() and all the VT-x/AMD-V specific bits
780 * should move into their respective modules. */
781 /* Finally, call global VT-x/AMD-V termination. */
782 if (g_fHmVmxSupported)
783 VMXR0GlobalTerm();
784 else if (g_fHmSvmSupported)
785 SVMR0GlobalTerm();
786
787 return rc;
788}
789
790
791/**
792 * Enable VT-x or AMD-V on the current CPU
793 *
794 * @returns VBox status code.
795 * @param pVM The cross context VM structure. Can be NULL.
796 * @param idCpu The identifier for the CPU the function is called on.
797 *
798 * @remarks Maybe called with interrupts disabled!
799 */
800static int hmR0EnableCpu(PVMCC pVM, RTCPUID idCpu)
801{
802 PHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
803
804 Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
805 Assert(idCpu < RT_ELEMENTS(g_aHmCpuInfo));
806 Assert(!pHostCpu->fConfigured);
807 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
808
809 pHostCpu->idCpu = idCpu;
810 /* Do NOT reset cTlbFlushes here, see @bugref{6255}. */
811
812 int rc;
813 if ( g_fHmVmxSupported
814 && g_fHmVmxUsingSUPR0EnableVTx)
815 rc = g_HmR0Ops.pfnEnableCpu(pHostCpu, pVM, NULL /* pvCpuPage */, NIL_RTHCPHYS, true, &g_HmMsrs);
816 else
817 {
818 AssertLogRelMsgReturn(pHostCpu->hMemObj != NIL_RTR0MEMOBJ, ("hmR0EnableCpu failed idCpu=%u.\n", idCpu), VERR_HM_IPE_1);
819 rc = g_HmR0Ops.pfnEnableCpu(pHostCpu, pVM, pHostCpu->pvMemObj, pHostCpu->HCPhysMemObj, false, &g_HmMsrs);
820 }
821 if (RT_SUCCESS(rc))
822 pHostCpu->fConfigured = true;
823 return rc;
824}
825
826
827/**
828 * Worker function passed to RTMpOnAll() that is to be called on all CPUs.
829 *
830 * @param idCpu The identifier for the CPU the function is called on.
831 * @param pvUser1 Opaque pointer to the VM (can be NULL!).
832 * @param pvUser2 The 2nd user argument.
833 */
834static DECLCALLBACK(void) hmR0EnableCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
835{
836 PVMCC pVM = (PVMCC)pvUser1; /* can be NULL! */
837 PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser2;
838 AssertReturnVoid(g_fHmGlobalInit);
839 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
840 hmR0FirstRcSetStatus(pFirstRc, hmR0EnableCpu(pVM, idCpu));
841}
842
843
844/**
845 * RTOnce callback employed by HMR0EnableAllCpus.
846 *
847 * @returns VBox status code.
848 * @param pvUser Pointer to the VM.
849 */
850static DECLCALLBACK(int32_t) hmR0EnableAllCpuOnce(void *pvUser)
851{
852 PVMCC pVM = (PVMCC)pvUser;
853
854 /*
855 * Indicate that we've initialized.
856 *
857 * Note! There is a potential race between this function and the suspend
858 * notification. Kind of unlikely though, so ignored for now.
859 */
860 AssertReturn(!g_fHmEnabled, VERR_HM_ALREADY_ENABLED_IPE);
861 ASMAtomicWriteBool(&g_fHmEnabled, true);
862
863 /*
864 * The global init variable is set by the first VM.
865 */
866 g_fHmGlobalInit = pVM->hm.s.fGlobalInit;
867
868#ifdef VBOX_STRICT
869 for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
870 {
871 Assert(g_aHmCpuInfo[i].hMemObj == NIL_RTR0MEMOBJ);
872 Assert(g_aHmCpuInfo[i].HCPhysMemObj == NIL_RTHCPHYS);
873 Assert(g_aHmCpuInfo[i].pvMemObj == NULL);
874 Assert(!g_aHmCpuInfo[i].fConfigured);
875 Assert(!g_aHmCpuInfo[i].cTlbFlushes);
876 Assert(!g_aHmCpuInfo[i].uCurrentAsid);
877# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
878 Assert(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm == NIL_RTR0MEMOBJ);
879 Assert(g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm == NIL_RTHCPHYS);
880 Assert(g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm == NULL);
881# endif
882 }
883#endif
884
885 int rc;
886 if ( g_fHmVmxSupported
887 && g_fHmVmxUsingSUPR0EnableVTx)
888 {
889 /*
890 * Global VT-x initialization API (only darwin for now).
891 */
892 rc = SUPR0EnableVTx(true /* fEnable */);
893 if (RT_SUCCESS(rc))
894 {
895 g_fHmVmxCalledSUPR0EnableVTx = true;
896 /* If the host provides a VT-x init API, then we'll rely on that for global init. */
897 g_fHmGlobalInit = pVM->hm.s.fGlobalInit = true;
898 }
899 else
900 AssertMsgFailed(("hmR0EnableAllCpuOnce/SUPR0EnableVTx: rc=%Rrc\n", rc));
901 }
902 else
903 {
904 /*
905 * We're doing the job ourselves.
906 */
907 /* Allocate one page per cpu for the global VT-x and AMD-V pages */
908 for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
909 {
910 Assert(g_aHmCpuInfo[i].hMemObj == NIL_RTR0MEMOBJ);
911#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
912 Assert(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm == NIL_RTR0MEMOBJ);
913#endif
914 if (RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(i)))
915 {
916 /** @todo NUMA */
917 rc = RTR0MemObjAllocCont(&g_aHmCpuInfo[i].hMemObj, HOST_PAGE_SIZE, false /* executable R0 mapping */);
918 AssertLogRelRCReturn(rc, rc);
919
920 g_aHmCpuInfo[i].HCPhysMemObj = RTR0MemObjGetPagePhysAddr(g_aHmCpuInfo[i].hMemObj, 0);
921 Assert(g_aHmCpuInfo[i].HCPhysMemObj != NIL_RTHCPHYS);
922 Assert(!(g_aHmCpuInfo[i].HCPhysMemObj & HOST_PAGE_OFFSET_MASK));
923
924 g_aHmCpuInfo[i].pvMemObj = RTR0MemObjAddress(g_aHmCpuInfo[i].hMemObj);
925 AssertPtr(g_aHmCpuInfo[i].pvMemObj);
926 RT_BZERO(g_aHmCpuInfo[i].pvMemObj, HOST_PAGE_SIZE);
927
928#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
929 rc = RTR0MemObjAllocCont(&g_aHmCpuInfo[i].n.svm.hNstGstMsrpm, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
930 false /* executable R0 mapping */);
931 AssertLogRelRCReturn(rc, rc);
932
933 g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm = RTR0MemObjGetPagePhysAddr(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm, 0);
934 Assert(g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm != NIL_RTHCPHYS);
935 Assert(!(g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm & HOST_PAGE_OFFSET_MASK));
936
937 g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm = RTR0MemObjAddress(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm);
938 AssertPtr(g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm);
939 ASMMemFill32(g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
940#endif
941 }
942 }
943
944 rc = VINF_SUCCESS;
945 }
946
947 if ( RT_SUCCESS(rc)
948 && g_fHmGlobalInit)
949 {
950 /* First time, so initialize each cpu/core. */
951 HMR0FIRSTRC FirstRc;
952 hmR0FirstRcInit(&FirstRc);
953 rc = RTMpOnAll(hmR0EnableCpuCallback, (void *)pVM, &FirstRc);
954 if (RT_SUCCESS(rc))
955 rc = hmR0FirstRcGetStatus(&FirstRc);
956 }
957
958 return rc;
959}
960
961
962/**
963 * Sets up HM on all cpus.
964 *
965 * @returns VBox status code.
966 * @param pVM The cross context VM structure.
967 */
968VMMR0_INT_DECL(int) HMR0EnableAllCpus(PVMCC pVM)
969{
970 /* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
971 if (ASMAtomicReadBool(&g_fHmSuspended))
972 return VERR_HM_SUSPEND_PENDING;
973
974 return RTOnce(&g_HmEnableAllCpusOnce, hmR0EnableAllCpuOnce, pVM);
975}
976
977
978/**
979 * Disable VT-x or AMD-V on the current CPU.
980 *
981 * @returns VBox status code.
982 * @param idCpu The identifier for the CPU this function is called on.
983 *
984 * @remarks Must be called with preemption disabled.
985 */
986static int hmR0DisableCpu(RTCPUID idCpu)
987{
988 PHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
989
990 Assert(!g_fHmVmxSupported || !g_fHmVmxUsingSUPR0EnableVTx);
991 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
992 Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
993 Assert(idCpu < RT_ELEMENTS(g_aHmCpuInfo));
994 Assert(!pHostCpu->fConfigured || pHostCpu->hMemObj != NIL_RTR0MEMOBJ);
995 AssertRelease(idCpu == RTMpCpuId());
996
997 if (pHostCpu->hMemObj == NIL_RTR0MEMOBJ)
998 return pHostCpu->fConfigured ? VERR_NO_MEMORY : VINF_SUCCESS /* not initialized. */;
999 AssertPtr(pHostCpu->pvMemObj);
1000 Assert(pHostCpu->HCPhysMemObj != NIL_RTHCPHYS);
1001
1002 int rc;
1003 if (pHostCpu->fConfigured)
1004 {
1005 rc = g_HmR0Ops.pfnDisableCpu(pHostCpu, pHostCpu->pvMemObj, pHostCpu->HCPhysMemObj);
1006 AssertRCReturn(rc, rc);
1007
1008 pHostCpu->fConfigured = false;
1009 pHostCpu->idCpu = NIL_RTCPUID;
1010 }
1011 else
1012 rc = VINF_SUCCESS; /* nothing to do */
1013 return rc;
1014}
1015
1016
1017/**
1018 * Worker function passed to RTMpOnAll() that is to be called on the target
1019 * CPUs.
1020 *
1021 * @param idCpu The identifier for the CPU the function is called on.
1022 * @param pvUser1 The 1st user argument.
1023 * @param pvUser2 Opaque pointer to the FirstRc.
1024 */
1025static DECLCALLBACK(void) hmR0DisableCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
1026{
1027 PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser2; NOREF(pvUser1);
1028 AssertReturnVoid(g_fHmGlobalInit);
1029 hmR0FirstRcSetStatus(pFirstRc, hmR0DisableCpu(idCpu));
1030}
1031
1032
1033/**
1034 * Worker function passed to RTMpOnSpecific() that is to be called on the target
1035 * CPU.
1036 *
1037 * @param idCpu The identifier for the CPU the function is called on.
1038 * @param pvUser1 Null, not used.
1039 * @param pvUser2 Null, not used.
1040 */
1041static DECLCALLBACK(void) hmR0DisableCpuOnSpecificCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
1042{
1043 NOREF(pvUser1);
1044 NOREF(pvUser2);
1045 hmR0DisableCpu(idCpu);
1046}
1047
1048
1049/**
1050 * Callback function invoked when a cpu goes online or offline.
1051 *
1052 * @param enmEvent The Mp event.
1053 * @param idCpu The identifier for the CPU the function is called on.
1054 * @param pvData Opaque data (PVMCC pointer).
1055 */
1056static DECLCALLBACK(void) hmR0MpEventCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData)
1057{
1058 NOREF(pvData);
1059 Assert(!g_fHmVmxSupported || !g_fHmVmxUsingSUPR0EnableVTx);
1060
1061 /*
1062 * We only care about uninitializing a CPU that is going offline. When a
1063 * CPU comes online, the initialization is done lazily in HMR0Enter().
1064 */
1065 switch (enmEvent)
1066 {
1067 case RTMPEVENT_OFFLINE:
1068 {
1069 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1070 RTThreadPreemptDisable(&PreemptState);
1071 if (idCpu == RTMpCpuId())
1072 {
1073 int rc = hmR0DisableCpu(idCpu);
1074 AssertRC(rc);
1075 RTThreadPreemptRestore(&PreemptState);
1076 }
1077 else
1078 {
1079 RTThreadPreemptRestore(&PreemptState);
1080 RTMpOnSpecific(idCpu, hmR0DisableCpuOnSpecificCallback, NULL /* pvUser1 */, NULL /* pvUser2 */);
1081 }
1082 break;
1083 }
1084
1085 default:
1086 break;
1087 }
1088}
1089
1090
1091/**
1092 * Called whenever a system power state change occurs.
1093 *
1094 * @param enmEvent The Power event.
1095 * @param pvUser User argument.
1096 */
1097static DECLCALLBACK(void) hmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser)
1098{
1099 NOREF(pvUser);
1100 Assert(!g_fHmVmxSupported || !g_fHmVmxUsingSUPR0EnableVTx);
1101
1102#ifdef LOG_ENABLED
1103 if (enmEvent == RTPOWEREVENT_SUSPEND)
1104 SUPR0Printf("hmR0PowerCallback RTPOWEREVENT_SUSPEND\n");
1105 else
1106 SUPR0Printf("hmR0PowerCallback RTPOWEREVENT_RESUME\n");
1107#endif
1108
1109 if (enmEvent == RTPOWEREVENT_SUSPEND)
1110 ASMAtomicWriteBool(&g_fHmSuspended, true);
1111
1112 if (g_fHmEnabled)
1113 {
1114 int rc;
1115 HMR0FIRSTRC FirstRc;
1116 hmR0FirstRcInit(&FirstRc);
1117
1118 if (enmEvent == RTPOWEREVENT_SUSPEND)
1119 {
1120 if (g_fHmGlobalInit)
1121 {
1122 /* Turn off VT-x or AMD-V on all CPUs. */
1123 rc = RTMpOnAll(hmR0DisableCpuCallback, NULL /* pvUser 1 */, &FirstRc);
1124 Assert(RT_SUCCESS(rc) || rc == VERR_NOT_SUPPORTED);
1125 }
1126 /* else nothing to do here for the local init case */
1127 }
1128 else
1129 {
1130 /* Reinit the CPUs from scratch as the suspend state might have
1131 messed with the MSRs. (lousy BIOSes as usual) */
1132 if (g_fHmVmxSupported)
1133 rc = RTMpOnAll(hmR0InitIntelCpu, &FirstRc, NULL);
1134 else
1135 rc = RTMpOnAll(hmR0InitAmdCpu, &FirstRc, NULL);
1136 Assert(RT_SUCCESS(rc) || rc == VERR_NOT_SUPPORTED);
1137 if (RT_SUCCESS(rc))
1138 rc = hmR0FirstRcGetStatus(&FirstRc);
1139#ifdef LOG_ENABLED
1140 if (RT_FAILURE(rc))
1141 SUPR0Printf("hmR0PowerCallback hmR0InitXxxCpu failed with %Rc\n", rc);
1142#endif
1143 if (g_fHmGlobalInit)
1144 {
1145 /* Turn VT-x or AMD-V back on on all CPUs. */
1146 rc = RTMpOnAll(hmR0EnableCpuCallback, NULL /* pVM */, &FirstRc /* output ignored */);
1147 Assert(RT_SUCCESS(rc) || rc == VERR_NOT_SUPPORTED);
1148 }
1149 /* else nothing to do here for the local init case */
1150 }
1151 }
1152
1153 if (enmEvent == RTPOWEREVENT_RESUME)
1154 ASMAtomicWriteBool(&g_fHmSuspended, false);
1155}
1156
1157
1158/**
1159 * Does ring-0 per-VM HM initialization.
1160 *
1161 * This will call the CPU specific init. routine which may initialize and allocate
1162 * resources for virtual CPUs.
1163 *
1164 * @returns VBox status code.
1165 * @param pVM The cross context VM structure.
1166 *
1167 * @remarks This is called after HMR3Init(), see vmR3CreateU() and
1168 * vmR3InitRing3().
1169 */
1170VMMR0_INT_DECL(int) HMR0InitVM(PVMCC pVM)
1171{
1172 AssertCompile(sizeof(pVM->hm.s) <= sizeof(pVM->hm.padding));
1173 AssertCompile(sizeof(pVM->hmr0.s) <= sizeof(pVM->hmr0.padding));
1174 AssertCompile(sizeof(pVM->aCpus[0].hm.s) <= sizeof(pVM->aCpus[0].hm.padding));
1175 AssertCompile(sizeof(pVM->aCpus[0].hmr0.s) <= sizeof(pVM->aCpus[0].hmr0.padding));
1176 AssertReturn(pVM, VERR_INVALID_PARAMETER);
1177
1178 /* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
1179 if (ASMAtomicReadBool(&g_fHmSuspended))
1180 return VERR_HM_SUSPEND_PENDING;
1181
1182 /*
1183 * Copy globals to the VM structure.
1184 */
1185 Assert(!(pVM->hm.s.vmx.fSupported && pVM->hm.s.svm.fSupported));
1186 if (pVM->hm.s.vmx.fSupported)
1187 {
1188 pVM->hmr0.s.vmx.fUsePreemptTimer = pVM->hm.s.vmx.fUsePreemptTimerCfg && g_fHmVmxUsePreemptTimer;
1189 pVM->hm.s.vmx.fUsePreemptTimerCfg = pVM->hmr0.s.vmx.fUsePreemptTimer;
1190 pVM->hm.s.vmx.cPreemptTimerShift = g_cHmVmxPreemptTimerShift;
1191 pVM->hm.s.ForR3.vmx.u64HostCr4 = g_uHmVmxHostCr4;
1192 pVM->hm.s.ForR3.vmx.u64HostMsrEfer = g_uHmVmxHostMsrEfer;
1193 pVM->hm.s.ForR3.vmx.u64HostSmmMonitorCtl = g_uHmVmxHostSmmMonitorCtl;
1194 pVM->hm.s.ForR3.vmx.u64HostFeatCtrl = g_HmMsrs.u.vmx.u64FeatCtrl;
1195 HMGetVmxMsrsFromHwvirtMsrs(&g_HmMsrs, &pVM->hm.s.ForR3.vmx.Msrs);
1196 /* If you need to tweak host MSRs for testing VMX R0 code, do it here. */
1197
1198 /* Enable VPID if supported and configured. */
1199 if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VPID)
1200 pVM->hm.s.ForR3.vmx.fVpid = pVM->hmr0.s.vmx.fVpid = pVM->hm.s.vmx.fAllowVpid; /* Can be overridden by CFGM in HMR3Init(). */
1201
1202 /* Use VMCS shadowing if supported. */
1203 pVM->hmr0.s.vmx.fUseVmcsShadowing = pVM->cpum.ro.GuestFeatures.fVmx
1204 && (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VMCS_SHADOWING);
1205 pVM->hm.s.ForR3.vmx.fUseVmcsShadowing = pVM->hmr0.s.vmx.fUseVmcsShadowing;
1206
1207 /* Use the VMCS controls for swapping the EFER MSR if supported. */
1208 pVM->hm.s.ForR3.vmx.fSupportsVmcsEfer = g_fHmVmxSupportsVmcsEfer;
1209
1210#if 0
1211 /* Enable APIC register virtualization and virtual-interrupt delivery if supported. */
1212 if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT)
1213 && (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY))
1214 pVM->hm.s.fVirtApicRegs = true;
1215
1216 /* Enable posted-interrupt processing if supported. */
1217 /** @todo Add and query IPRT API for host OS support for posted-interrupt IPI
1218 * here. */
1219 if ( (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT)
1220 && (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT))
1221 pVM->hm.s.fPostedIntrs = true;
1222#endif
1223 }
1224 else if (pVM->hm.s.svm.fSupported)
1225 {
1226 pVM->hm.s.ForR3.svm.u32Rev = g_uHmSvmRev;
1227 pVM->hm.s.ForR3.svm.fFeatures = g_fHmSvmFeatures;
1228 pVM->hm.s.ForR3.svm.u64MsrHwcr = g_HmMsrs.u.svm.u64MsrHwcr;
1229 /* If you need to tweak host MSRs for testing SVM R0 code, do it here. */
1230 }
1231 pVM->hm.s.ForR3.rcInit = g_rcHmInit;
1232 pVM->hm.s.ForR3.uMaxAsid = g_uHmMaxAsid;
1233
1234 /*
1235 * Set default maximum inner loops in ring-0 before returning to ring-3.
1236 * Can be overriden using CFGM.
1237 */
1238 uint32_t cMaxResumeLoops = pVM->hm.s.cMaxResumeLoopsCfg;
1239 if (!cMaxResumeLoops)
1240 {
1241 cMaxResumeLoops = 1024;
1242 if (RTThreadPreemptIsPendingTrusty())
1243 cMaxResumeLoops = 8192;
1244 }
1245 else if (cMaxResumeLoops > 16384)
1246 cMaxResumeLoops = 16384;
1247 else if (cMaxResumeLoops < 32)
1248 cMaxResumeLoops = 32;
1249 pVM->hm.s.cMaxResumeLoopsCfg = pVM->hmr0.s.cMaxResumeLoops = cMaxResumeLoops;
1250
1251 /*
1252 * Initialize some per-VCPU fields.
1253 */
1254 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1255 {
1256 PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1257 pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID;
1258 pVCpu->hmr0.s.idLastCpu = NIL_RTCPUID;
1259
1260 /* We'll aways increment this the first time (host uses ASID 0). */
1261 AssertReturn(!pVCpu->hmr0.s.uCurrentAsid, VERR_HM_IPE_3);
1262 }
1263
1264 /*
1265 * Configure defences against spectre and other CPU bugs.
1266 */
1267 uint32_t fWorldSwitcher = 0;
1268 uint32_t cLastStdLeaf = ASMCpuId_EAX(0);
1269 if (cLastStdLeaf >= 0x00000007 && RTX86IsValidStdRange(cLastStdLeaf))
1270 {
1271 uint32_t uEdx = 0;
1272 ASMCpuIdExSlow(0x00000007, 0, 0, 0, NULL, NULL, NULL, &uEdx);
1273
1274 if (uEdx & X86_CPUID_STEXT_FEATURE_EDX_IBRS_IBPB)
1275 {
1276 if (pVM->hm.s.fIbpbOnVmExit)
1277 fWorldSwitcher |= HM_WSF_IBPB_EXIT;
1278 if (pVM->hm.s.fIbpbOnVmEntry)
1279 fWorldSwitcher |= HM_WSF_IBPB_ENTRY;
1280 }
1281 if (uEdx & X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD)
1282 {
1283 if (pVM->hm.s.fL1dFlushOnVmEntry)
1284 fWorldSwitcher |= HM_WSF_L1D_ENTRY;
1285 else if (pVM->hm.s.fL1dFlushOnSched)
1286 fWorldSwitcher |= HM_WSF_L1D_SCHED;
1287 }
1288 if (uEdx & X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR)
1289 {
1290 if (pVM->hm.s.fMdsClearOnVmEntry)
1291 fWorldSwitcher |= HM_WSF_MDS_ENTRY;
1292 else if (pVM->hm.s.fMdsClearOnSched)
1293 fWorldSwitcher |= HM_WSF_MDS_SCHED;
1294 }
1295 }
1296 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1297 {
1298 PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1299 pVCpu->hmr0.s.fWorldSwitcher = fWorldSwitcher;
1300 }
1301 pVM->hm.s.ForR3.fWorldSwitcher = fWorldSwitcher;
1302
1303
1304 /*
1305 * Call the hardware specific initialization method.
1306 */
1307 return g_HmR0Ops.pfnInitVM(pVM);
1308}
1309
1310
1311/**
1312 * Does ring-0 per VM HM termination.
1313 *
1314 * @returns VBox status code.
1315 * @param pVM The cross context VM structure.
1316 */
1317VMMR0_INT_DECL(int) HMR0TermVM(PVMCC pVM)
1318{
1319 Log(("HMR0TermVM: %p\n", pVM));
1320 AssertReturn(pVM, VERR_INVALID_PARAMETER);
1321
1322 /*
1323 * Call the hardware specific method.
1324 *
1325 * Note! We might be preparing for a suspend, so the pfnTermVM() functions should probably not
1326 * mess with VT-x/AMD-V features on the CPU, currently all they do is free memory so this is safe.
1327 */
1328 return g_HmR0Ops.pfnTermVM(pVM);
1329}
1330
1331
1332/**
1333 * Sets up a VT-x or AMD-V session.
1334 *
1335 * This is mostly about setting up the hardware VM state.
1336 *
1337 * @returns VBox status code.
1338 * @param pVM The cross context VM structure.
1339 */
1340VMMR0_INT_DECL(int) HMR0SetupVM(PVMCC pVM)
1341{
1342 Log(("HMR0SetupVM: %p\n", pVM));
1343 AssertReturn(pVM, VERR_INVALID_PARAMETER);
1344
1345 /* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
1346 AssertReturn(!ASMAtomicReadBool(&g_fHmSuspended), VERR_HM_SUSPEND_PENDING);
1347
1348 /* On first entry we'll sync everything. */
1349 VMCC_FOR_EACH_VMCPU_STMT(pVM, pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_ALL_GUEST);
1350
1351 /*
1352 * Call the hardware specific setup VM method. This requires the CPU to be
1353 * enabled for AMD-V/VT-x and preemption to be prevented.
1354 */
1355 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1356 RTThreadPreemptDisable(&PreemptState);
1357 RTCPUID const idCpu = RTMpCpuId();
1358
1359 /* Enable VT-x or AMD-V if local init is required. */
1360 int rc;
1361 if (!g_fHmGlobalInit)
1362 {
1363 Assert(!g_fHmVmxSupported || !g_fHmVmxUsingSUPR0EnableVTx);
1364 rc = hmR0EnableCpu(pVM, idCpu);
1365 if (RT_FAILURE(rc))
1366 {
1367 RTThreadPreemptRestore(&PreemptState);
1368 return rc;
1369 }
1370 }
1371
1372 /* Setup VT-x or AMD-V. */
1373 rc = g_HmR0Ops.pfnSetupVM(pVM);
1374
1375 /* Disable VT-x or AMD-V if local init was done before. */
1376 if (!g_fHmGlobalInit)
1377 {
1378 Assert(!g_fHmVmxSupported || !g_fHmVmxUsingSUPR0EnableVTx);
1379 int rc2 = hmR0DisableCpu(idCpu);
1380 AssertRC(rc2);
1381 }
1382
1383 RTThreadPreemptRestore(&PreemptState);
1384 return rc;
1385}
1386
1387
1388/**
1389 * Notification callback before an assertion longjump and guru mediation.
1390 *
1391 * @returns VBox status code.
1392 * @param pVCpu The cross context virtual CPU structure.
1393 * @param pvUser User argument, currently unused, NULL.
1394 */
1395static DECLCALLBACK(int) hmR0AssertionCallback(PVMCPUCC pVCpu, void *pvUser)
1396{
1397 RT_NOREF(pvUser);
1398 Assert(pVCpu);
1399 Assert(g_HmR0Ops.pfnAssertionCallback);
1400 return g_HmR0Ops.pfnAssertionCallback(pVCpu);
1401}
1402
1403
1404/**
1405 * Turns on HM on the CPU if necessary and initializes the bare minimum state
1406 * required for entering HM context.
1407 *
1408 * @returns VBox status code.
1409 * @param pVCpu The cross context virtual CPU structure.
1410 *
1411 * @remarks No-long-jump zone!!!
1412 */
1413VMMR0_INT_DECL(int) hmR0EnterCpu(PVMCPUCC pVCpu)
1414{
1415 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1416
1417 int rc = VINF_SUCCESS;
1418 RTCPUID const idCpu = RTMpCpuId();
1419 PHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
1420 AssertPtr(pHostCpu);
1421
1422 /* Enable VT-x or AMD-V if local init is required, or enable if it's a freshly onlined CPU. */
1423 if (!pHostCpu->fConfigured)
1424 rc = hmR0EnableCpu(pVCpu->CTX_SUFF(pVM), idCpu);
1425
1426 /* Register a callback to fire prior to performing a longjmp to ring-3 so HM can disable VT-x/AMD-V if needed. */
1427 VMMR0AssertionSetNotification(pVCpu, hmR0AssertionCallback, NULL /*pvUser*/);
1428
1429 /* Reload host-state (back from ring-3/migrated CPUs) and shared guest/host bits. */
1430 if (g_fHmVmxSupported)
1431 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
1432 else
1433 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE;
1434
1435 Assert(pHostCpu->idCpu == idCpu && pHostCpu->idCpu != NIL_RTCPUID);
1436 pVCpu->hmr0.s.idEnteredCpu = idCpu;
1437 return rc;
1438}
1439
1440
1441/**
1442 * Enters the VT-x or AMD-V session.
1443 *
1444 * @returns VBox status code.
1445 * @param pVCpu The cross context virtual CPU structure.
1446 *
1447 * @remarks This is called with preemption disabled.
1448 */
1449VMMR0_INT_DECL(int) HMR0Enter(PVMCPUCC pVCpu)
1450{
1451 /* Make sure we can't enter a session after we've disabled HM in preparation of a suspend. */
1452 AssertReturn(!ASMAtomicReadBool(&g_fHmSuspended), VERR_HM_SUSPEND_PENDING);
1453 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1454
1455 /* Load the bare minimum state required for entering HM. */
1456 int rc = hmR0EnterCpu(pVCpu);
1457 if (RT_SUCCESS(rc))
1458 {
1459 if (g_fHmVmxSupported)
1460 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
1461 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
1462 else
1463 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
1464 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
1465
1466 /* Keep track of the CPU owning the VMCS for debugging scheduling weirdness and ring-3 calls. */
1467 rc = g_HmR0Ops.pfnEnterSession(pVCpu);
1468 AssertMsgRCReturnStmt(rc, ("rc=%Rrc pVCpu=%p\n", rc, pVCpu), pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID, rc);
1469
1470 /* Exports the host-state as we may be resuming code after a longjmp and quite
1471 possibly now be scheduled on a different CPU. */
1472 rc = g_HmR0Ops.pfnExportHostState(pVCpu);
1473 AssertMsgRCReturnStmt(rc, ("rc=%Rrc pVCpu=%p\n", rc, pVCpu), pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID, rc);
1474 }
1475 return rc;
1476}
1477
1478
1479/**
1480 * Deinitializes the bare minimum state used for HM context and if necessary
1481 * disable HM on the CPU.
1482 *
1483 * @returns VBox status code.
1484 * @param pVCpu The cross context virtual CPU structure.
1485 *
1486 * @remarks No-long-jump zone!!!
1487 */
1488VMMR0_INT_DECL(int) HMR0LeaveCpu(PVMCPUCC pVCpu)
1489{
1490 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1491 VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_HM_WRONG_CPU);
1492
1493 RTCPUID const idCpu = RTMpCpuId();
1494 PCHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
1495
1496 if ( !g_fHmGlobalInit
1497 && pHostCpu->fConfigured)
1498 {
1499 int rc = hmR0DisableCpu(idCpu);
1500 AssertRCReturn(rc, rc);
1501 Assert(!pHostCpu->fConfigured);
1502 Assert(pHostCpu->idCpu == NIL_RTCPUID);
1503
1504 /* For obtaining a non-zero ASID/VPID on next re-entry. */
1505 pVCpu->hmr0.s.idLastCpu = NIL_RTCPUID;
1506 }
1507
1508 /* Clear it while leaving HM context, hmPokeCpuForTlbFlush() relies on this. */
1509 pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID;
1510
1511 /* De-register the longjmp-to-ring 3 callback now that we have reliquished hardware resources. */
1512 VMMR0AssertionRemoveNotification(pVCpu);
1513 return VINF_SUCCESS;
1514}
1515
1516
1517/**
1518 * Thread-context hook for HM.
1519 *
1520 * This is used together with RTThreadCtxHookCreate() on platforms which
1521 * supports it, and directly from VMMR0EmtPrepareForBlocking() and
1522 * VMMR0EmtResumeAfterBlocking() on platforms which don't.
1523 *
1524 * @param enmEvent The thread-context event.
1525 * @param pvUser Opaque pointer to the VMCPU.
1526 */
1527VMMR0_INT_DECL(void) HMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
1528{
1529 PVMCPUCC pVCpu = (PVMCPUCC)pvUser;
1530 Assert(pVCpu);
1531 Assert(g_HmR0Ops.pfnThreadCtxCallback);
1532
1533 g_HmR0Ops.pfnThreadCtxCallback(enmEvent, pVCpu, g_fHmGlobalInit);
1534}
1535
1536
1537/**
1538 * Runs guest code in a hardware accelerated VM.
1539 *
1540 * @returns Strict VBox status code. (VBOXSTRICTRC isn't used because it's
1541 * called from setjmp assembly.)
1542 * @param pVM The cross context VM structure.
1543 * @param pVCpu The cross context virtual CPU structure.
1544 *
1545 * @remarks Can be called with preemption enabled if thread-context hooks are
1546 * used!!!
1547 */
1548VMMR0_INT_DECL(int) HMR0RunGuestCode(PVMCC pVM, PVMCPUCC pVCpu)
1549{
1550 RT_NOREF(pVM);
1551
1552#ifdef VBOX_STRICT
1553 /* With thread-context hooks we would be running this code with preemption enabled. */
1554 if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
1555 {
1556 PCHMPHYSCPU pHostCpu = &g_aHmCpuInfo[RTMpCpuId()];
1557 Assert(!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1558 Assert(pHostCpu->fConfigured);
1559 AssertReturn(!ASMAtomicReadBool(&g_fHmSuspended), VERR_HM_SUSPEND_PENDING);
1560 }
1561#endif
1562
1563 VBOXSTRICTRC rcStrict = g_HmR0Ops.pfnRunGuestCode(pVCpu);
1564 return VBOXSTRICTRC_VAL(rcStrict);
1565}
1566
1567
1568/**
1569 * Notification from CPUM that it has unloaded the guest FPU/SSE/AVX state from
1570 * the host CPU and that guest access to it must be intercepted.
1571 *
1572 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1573 */
1574VMMR0_INT_DECL(void) HMR0NotifyCpumUnloadedGuestFpuState(PVMCPUCC pVCpu)
1575{
1576 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
1577}
1578
1579
1580/**
1581 * Notification from CPUM that it has modified the host CR0 (because of FPU).
1582 *
1583 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1584 */
1585VMMR0_INT_DECL(void) HMR0NotifyCpumModifiedHostCr0(PVMCPUCC pVCpu)
1586{
1587 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT);
1588}
1589
1590
1591/**
1592 * Returns suspend status of the host.
1593 *
1594 * @returns Suspend pending or not.
1595 */
1596VMMR0_INT_DECL(bool) HMR0SuspendPending(void)
1597{
1598 return ASMAtomicReadBool(&g_fHmSuspended);
1599}
1600
1601
1602/**
1603 * Invalidates a guest page from the host TLB.
1604 *
1605 * @param pVCpu The cross context virtual CPU structure.
1606 * @param GCVirt Page to invalidate.
1607 */
1608VMMR0_INT_DECL(int) HMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1609{
1610 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1611 if (pVM->hm.s.vmx.fSupported)
1612 return VMXR0InvalidatePage(pVCpu, GCVirt);
1613 return SVMR0InvalidatePage(pVCpu, GCVirt);
1614}
1615
1616
1617/**
1618 * Returns the cpu structure for the current cpu.
1619 * Keep in mind that there is no guarantee it will stay the same (long jumps to ring 3!!!).
1620 *
1621 * @returns The cpu structure pointer.
1622 */
1623VMMR0_INT_DECL(PHMPHYSCPU) hmR0GetCurrentCpu(void)
1624{
1625 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1626 RTCPUID const idCpu = RTMpCpuId();
1627 Assert(idCpu < RT_ELEMENTS(g_aHmCpuInfo));
1628 return &g_aHmCpuInfo[idCpu];
1629}
1630
1631
1632/**
1633 * Interface for importing state on demand (used by IEM).
1634 *
1635 * @returns VBox status code.
1636 * @param pVCpu The cross context CPU structure.
1637 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1638 */
1639VMMR0_INT_DECL(int) HMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
1640{
1641 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
1642 return VMXR0ImportStateOnDemand(pVCpu, fWhat);
1643 return SVMR0ImportStateOnDemand(pVCpu, fWhat);
1644}
1645
1646
1647/**
1648 * Gets HM VM-exit auxiliary information.
1649 *
1650 * @returns VBox status code.
1651 * @param pVCpu The cross context CPU structure.
1652 * @param pHmExitAux Where to store the auxiliary info.
1653 * @param fWhat What to get, see HMVMX_READ_XXX. This is ignored/unused
1654 * on AMD-V.
1655 *
1656 * @remarks Currently this works only when executing a nested-guest using
1657 * hardware-assisted execution as it's where the auxiliary information is
1658 * required outside of HM. In the future we can make this available while
1659 * executing a regular (non-nested) guest if necessary.
1660 */
1661VMMR0_INT_DECL(int) HMR0GetExitAuxInfo(PVMCPUCC pVCpu, PHMEXITAUX pHmExitAux, uint32_t fWhat)
1662{
1663 Assert(pHmExitAux);
1664 Assert(!(fWhat & ~HMVMX_READ_VALID_MASK));
1665 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
1666 return VMXR0GetExitAuxInfo(pVCpu, &pHmExitAux->Vmx, fWhat);
1667 return SVMR0GetExitAuxInfo(pVCpu, &pHmExitAux->Svm);
1668}
1669
1670
1671#ifdef VBOX_STRICT
1672
1673/**
1674 * Dumps a descriptor.
1675 *
1676 * @param pDesc Descriptor to dump.
1677 * @param Sel The selector.
1678 * @param pszSel The name of the selector.
1679 */
1680VMMR0_INT_DECL(void) hmR0DumpDescriptor(PCX86DESCHC pDesc, RTSEL Sel, const char *pszSel)
1681{
1682 /*
1683 * Make variable description string.
1684 */
1685 static struct
1686 {
1687 unsigned cch;
1688 const char *psz;
1689 } const s_aTypes[32] =
1690 {
1691# define STRENTRY(str) { sizeof(str) - 1, str }
1692
1693 /* system */
1694# if HC_ARCH_BITS == 64
1695 STRENTRY("Reserved0 "), /* 0x00 */
1696 STRENTRY("Reserved1 "), /* 0x01 */
1697 STRENTRY("LDT "), /* 0x02 */
1698 STRENTRY("Reserved3 "), /* 0x03 */
1699 STRENTRY("Reserved4 "), /* 0x04 */
1700 STRENTRY("Reserved5 "), /* 0x05 */
1701 STRENTRY("Reserved6 "), /* 0x06 */
1702 STRENTRY("Reserved7 "), /* 0x07 */
1703 STRENTRY("Reserved8 "), /* 0x08 */
1704 STRENTRY("TSS64Avail "), /* 0x09 */
1705 STRENTRY("ReservedA "), /* 0x0a */
1706 STRENTRY("TSS64Busy "), /* 0x0b */
1707 STRENTRY("Call64 "), /* 0x0c */
1708 STRENTRY("ReservedD "), /* 0x0d */
1709 STRENTRY("Int64 "), /* 0x0e */
1710 STRENTRY("Trap64 "), /* 0x0f */
1711# else
1712 STRENTRY("Reserved0 "), /* 0x00 */
1713 STRENTRY("TSS16Avail "), /* 0x01 */
1714 STRENTRY("LDT "), /* 0x02 */
1715 STRENTRY("TSS16Busy "), /* 0x03 */
1716 STRENTRY("Call16 "), /* 0x04 */
1717 STRENTRY("Task "), /* 0x05 */
1718 STRENTRY("Int16 "), /* 0x06 */
1719 STRENTRY("Trap16 "), /* 0x07 */
1720 STRENTRY("Reserved8 "), /* 0x08 */
1721 STRENTRY("TSS32Avail "), /* 0x09 */
1722 STRENTRY("ReservedA "), /* 0x0a */
1723 STRENTRY("TSS32Busy "), /* 0x0b */
1724 STRENTRY("Call32 "), /* 0x0c */
1725 STRENTRY("ReservedD "), /* 0x0d */
1726 STRENTRY("Int32 "), /* 0x0e */
1727 STRENTRY("Trap32 "), /* 0x0f */
1728# endif
1729 /* non system */
1730 STRENTRY("DataRO "), /* 0x10 */
1731 STRENTRY("DataRO Accessed "), /* 0x11 */
1732 STRENTRY("DataRW "), /* 0x12 */
1733 STRENTRY("DataRW Accessed "), /* 0x13 */
1734 STRENTRY("DataDownRO "), /* 0x14 */
1735 STRENTRY("DataDownRO Accessed "), /* 0x15 */
1736 STRENTRY("DataDownRW "), /* 0x16 */
1737 STRENTRY("DataDownRW Accessed "), /* 0x17 */
1738 STRENTRY("CodeEO "), /* 0x18 */
1739 STRENTRY("CodeEO Accessed "), /* 0x19 */
1740 STRENTRY("CodeER "), /* 0x1a */
1741 STRENTRY("CodeER Accessed "), /* 0x1b */
1742 STRENTRY("CodeConfEO "), /* 0x1c */
1743 STRENTRY("CodeConfEO Accessed "), /* 0x1d */
1744 STRENTRY("CodeConfER "), /* 0x1e */
1745 STRENTRY("CodeConfER Accessed ") /* 0x1f */
1746# undef SYSENTRY
1747 };
1748# define ADD_STR(psz, pszAdd) do { strcpy(psz, pszAdd); psz += strlen(pszAdd); } while (0)
1749 char szMsg[128];
1750 char *psz = &szMsg[0];
1751 unsigned i = pDesc->Gen.u1DescType << 4 | pDesc->Gen.u4Type;
1752 memcpy(psz, s_aTypes[i].psz, s_aTypes[i].cch);
1753 psz += s_aTypes[i].cch;
1754
1755 if (pDesc->Gen.u1Present)
1756 ADD_STR(psz, "Present ");
1757 else
1758 ADD_STR(psz, "Not-Present ");
1759# if HC_ARCH_BITS == 64
1760 if (pDesc->Gen.u1Long)
1761 ADD_STR(psz, "64-bit ");
1762 else
1763 ADD_STR(psz, "Comp ");
1764# else
1765 if (pDesc->Gen.u1Granularity)
1766 ADD_STR(psz, "Page ");
1767 if (pDesc->Gen.u1DefBig)
1768 ADD_STR(psz, "32-bit ");
1769 else
1770 ADD_STR(psz, "16-bit ");
1771# endif
1772# undef ADD_STR
1773 *psz = '\0';
1774
1775 /*
1776 * Limit and Base and format the output.
1777 */
1778#ifdef LOG_ENABLED
1779 uint32_t u32Limit = X86DESC_LIMIT_G(pDesc);
1780
1781# if HC_ARCH_BITS == 64
1782 uint64_t const u64Base = X86DESC64_BASE(pDesc);
1783 Log((" %s { %#04x - %#RX64 %#RX64 - base=%#RX64 limit=%#08x dpl=%d } %s\n", pszSel,
1784 Sel, pDesc->au64[0], pDesc->au64[1], u64Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1785# else
1786 uint32_t const u32Base = X86DESC_BASE(pDesc);
1787 Log((" %s { %#04x - %#08x %#08x - base=%#08x limit=%#08x dpl=%d } %s\n", pszSel,
1788 Sel, pDesc->au32[0], pDesc->au32[1], u32Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1789# endif
1790#else
1791 NOREF(Sel); NOREF(pszSel);
1792#endif
1793}
1794
1795
1796/**
1797 * Formats a full register dump.
1798 *
1799 * @param pVCpu The cross context virtual CPU structure.
1800 * @param fFlags The dumping flags (HM_DUMP_REG_FLAGS_XXX).
1801 */
1802VMMR0_INT_DECL(void) hmR0DumpRegs(PVMCPUCC pVCpu, uint32_t fFlags)
1803{
1804 /*
1805 * Format the flags.
1806 */
1807 static struct
1808 {
1809 const char *pszSet;
1810 const char *pszClear;
1811 uint32_t fFlag;
1812 } const s_aFlags[] =
1813 {
1814 { "vip", NULL, X86_EFL_VIP },
1815 { "vif", NULL, X86_EFL_VIF },
1816 { "ac", NULL, X86_EFL_AC },
1817 { "vm", NULL, X86_EFL_VM },
1818 { "rf", NULL, X86_EFL_RF },
1819 { "nt", NULL, X86_EFL_NT },
1820 { "ov", "nv", X86_EFL_OF },
1821 { "dn", "up", X86_EFL_DF },
1822 { "ei", "di", X86_EFL_IF },
1823 { "tf", NULL, X86_EFL_TF },
1824 { "nt", "pl", X86_EFL_SF },
1825 { "nz", "zr", X86_EFL_ZF },
1826 { "ac", "na", X86_EFL_AF },
1827 { "po", "pe", X86_EFL_PF },
1828 { "cy", "nc", X86_EFL_CF },
1829 };
1830 char szEFlags[80];
1831 char *psz = szEFlags;
1832 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1833 uint32_t fEFlags = pCtx->eflags.u;
1834 for (unsigned i = 0; i < RT_ELEMENTS(s_aFlags); i++)
1835 {
1836 const char *pszAdd = s_aFlags[i].fFlag & fEFlags ? s_aFlags[i].pszSet : s_aFlags[i].pszClear;
1837 if (pszAdd)
1838 {
1839 strcpy(psz, pszAdd);
1840 psz += strlen(pszAdd);
1841 *psz++ = ' ';
1842 }
1843 }
1844 psz[-1] = '\0';
1845
1846 if (fFlags & HM_DUMP_REG_FLAGS_GPRS)
1847 {
1848 /*
1849 * Format the registers.
1850 */
1851 if (CPUMIsGuestIn64BitCode(pVCpu))
1852 Log(("rax=%016RX64 rbx=%016RX64 rcx=%016RX64 rdx=%016RX64\n"
1853 "rsi=%016RX64 rdi=%016RX64 r8 =%016RX64 r9 =%016RX64\n"
1854 "r10=%016RX64 r11=%016RX64 r12=%016RX64 r13=%016RX64\n"
1855 "r14=%016RX64 r15=%016RX64\n"
1856 "rip=%016RX64 rsp=%016RX64 rbp=%016RX64 iopl=%d %*s\n"
1857 "cs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1858 "ds={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1859 "es={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1860 "fs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1861 "gs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1862 "ss={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1863 "cr0=%016RX64 cr2=%016RX64 cr3=%016RX64 cr4=%016RX64\n"
1864 "dr0=%016RX64 dr1=%016RX64 dr2=%016RX64 dr3=%016RX64\n"
1865 "dr4=%016RX64 dr5=%016RX64 dr6=%016RX64 dr7=%016RX64\n"
1866 "gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1867 "ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1868 "tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1869 "SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1870 ,
1871 pCtx->rax, pCtx->rbx, pCtx->rcx, pCtx->rdx, pCtx->rsi, pCtx->rdi,
1872 pCtx->r8, pCtx->r9, pCtx->r10, pCtx->r11, pCtx->r12, pCtx->r13,
1873 pCtx->r14, pCtx->r15,
1874 pCtx->rip, pCtx->rsp, pCtx->rbp, X86_EFL_GET_IOPL(fEFlags), 31, szEFlags,
1875 pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u,
1876 pCtx->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u,
1877 pCtx->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u,
1878 pCtx->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u,
1879 pCtx->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u,
1880 pCtx->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u,
1881 pCtx->cr0, pCtx->cr2, pCtx->cr3, pCtx->cr4,
1882 pCtx->dr[0], pCtx->dr[1], pCtx->dr[2], pCtx->dr[3],
1883 pCtx->dr[4], pCtx->dr[5], pCtx->dr[6], pCtx->dr[7],
1884 pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, fEFlags,
1885 pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
1886 pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
1887 pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1888 else
1889 Log(("eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
1890 "eip=%08x esp=%08x ebp=%08x iopl=%d %*s\n"
1891 "cs={%04x base=%016RX64 limit=%08x flags=%08x} dr0=%08RX64 dr1=%08RX64\n"
1892 "ds={%04x base=%016RX64 limit=%08x flags=%08x} dr2=%08RX64 dr3=%08RX64\n"
1893 "es={%04x base=%016RX64 limit=%08x flags=%08x} dr4=%08RX64 dr5=%08RX64\n"
1894 "fs={%04x base=%016RX64 limit=%08x flags=%08x} dr6=%08RX64 dr7=%08RX64\n"
1895 "gs={%04x base=%016RX64 limit=%08x flags=%08x} cr0=%08RX64 cr2=%08RX64\n"
1896 "ss={%04x base=%016RX64 limit=%08x flags=%08x} cr3=%08RX64 cr4=%08RX64\n"
1897 "gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1898 "ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1899 "tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1900 "SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1901 ,
1902 pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
1903 pCtx->eip, pCtx->esp, pCtx->ebp, X86_EFL_GET_IOPL(fEFlags), 31, szEFlags,
1904 pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u, pCtx->dr[0], pCtx->dr[1],
1905 pCtx->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u, pCtx->dr[2], pCtx->dr[3],
1906 pCtx->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u, pCtx->dr[4], pCtx->dr[5],
1907 pCtx->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u, pCtx->dr[6], pCtx->dr[7],
1908 pCtx->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u, pCtx->cr0, pCtx->cr2,
1909 pCtx->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u, pCtx->cr3, pCtx->cr4,
1910 pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, fEFlags,
1911 pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
1912 pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
1913 pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1914 }
1915
1916 if (fFlags & HM_DUMP_REG_FLAGS_FPU)
1917 {
1918 PCX86FXSTATE pFpuCtx = &pCtx->XState.x87;
1919 Log(("FPU:\n"
1920 "FCW=%04x FSW=%04x FTW=%02x\n"
1921 "FOP=%04x FPUIP=%08x CS=%04x Rsrvd1=%04x\n"
1922 "FPUDP=%04x DS=%04x Rsvrd2=%04x MXCSR=%08x MXCSR_MASK=%08x\n"
1923 ,
1924 pFpuCtx->FCW, pFpuCtx->FSW, pFpuCtx->FTW,
1925 pFpuCtx->FOP, pFpuCtx->FPUIP, pFpuCtx->CS, pFpuCtx->Rsrvd1,
1926 pFpuCtx->FPUDP, pFpuCtx->DS, pFpuCtx->Rsrvd2,
1927 pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK));
1928 NOREF(pFpuCtx);
1929 }
1930
1931 if (fFlags & HM_DUMP_REG_FLAGS_MSRS)
1932 Log(("MSR:\n"
1933 "EFER =%016RX64\n"
1934 "PAT =%016RX64\n"
1935 "STAR =%016RX64\n"
1936 "CSTAR =%016RX64\n"
1937 "LSTAR =%016RX64\n"
1938 "SFMASK =%016RX64\n"
1939 "KERNELGSBASE =%016RX64\n",
1940 pCtx->msrEFER,
1941 pCtx->msrPAT,
1942 pCtx->msrSTAR,
1943 pCtx->msrCSTAR,
1944 pCtx->msrLSTAR,
1945 pCtx->msrSFMASK,
1946 pCtx->msrKERNELGSBASE));
1947}
1948
1949#endif /* VBOX_STRICT */
1950
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use