; $Id: HMR0A.asm 101524 2023-10-20 14:47:42Z vboxsync $ ;; @file ; HM - Ring-0 VMX, SVM world-switch and helper routines. ; ; ; Copyright (C) 2006-2023 Oracle and/or its affiliates. ; ; This file is part of VirtualBox base platform packages, as ; available from https://www.virtualbox.org. ; ; This program is free software; you can redistribute it and/or ; modify it under the terms of the GNU General Public License ; as published by the Free Software Foundation, in version 3 of the ; License. ; ; This program is distributed in the hope that it will be useful, but ; WITHOUT ANY WARRANTY; without even the implied warranty of ; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ; General Public License for more details. ; ; You should have received a copy of the GNU General Public License ; along with this program; if not, see . ; ; SPDX-License-Identifier: GPL-3.0-only ; ;********************************************************************************************************************************* ;* Header Files * ;********************************************************************************************************************************* ;%define RT_ASM_WITH_SEH64 - trouble with SEH, alignment and (probably) 2nd pass optimizations. %define RT_ASM_WITH_SEH64_ALT ; Use asmdefs.mac hackery for manually emitting unwind info. %include "VBox/asmdefs.mac" %include "VBox/err.mac" %include "VBox/vmm/hm_vmx.mac" %include "VBox/vmm/cpum.mac" %include "VBox/vmm/gvm.mac" %include "iprt/x86.mac" %include "HMInternal.mac" %ifndef RT_ARCH_AMD64 %error AMD64 only. %endif ;********************************************************************************************************************************* ;* Defined Constants And Macros * ;********************************************************************************************************************************* ;; The offset of the XMM registers in X86FXSTATE. ; Use define because I'm too lazy to convert the struct. %define XMM_OFF_IN_X86FXSTATE 160 ;; Spectre filler for 64-bit mode. ; Choosen to be an invalid address (also with 5 level paging). %define SPECTRE_FILLER 0x02204204207fffff ;; ; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation. ; ; @note This is normally done by hmR0VmxExportHostSegmentRegs and VMXRestoreHostState, ; so much of this is untested code. ; @{ %define VMX_SKIP_GDTR %define VMX_SKIP_TR %define VBOX_SKIP_RESTORE_SEG %ifdef RT_OS_DARWIN ; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't ; risk loading a stale LDT value or something invalid. %define HM_64_BIT_USE_NULL_SEL ; Darwin (Mavericks) uses IDTR limit to store the CPU number so we need to always restore it. ; See @bugref{6875}. %undef VMX_SKIP_IDTR %else %define VMX_SKIP_IDTR %endif ;; @} ;; @def CALLEE_PRESERVED_REGISTER_COUNT ; Number of registers pushed by PUSH_CALLEE_PRESERVED_REGISTERS %ifdef ASM_CALL64_GCC %define CALLEE_PRESERVED_REGISTER_COUNT 5 %else %define CALLEE_PRESERVED_REGISTER_COUNT 7 %endif ;; @def PUSH_CALLEE_PRESERVED_REGISTERS ; Macro for pushing all GPRs we must preserve for the caller. %macro PUSH_CALLEE_PRESERVED_REGISTERS 0 push r15 SEH64_PUSH_GREG r15 %assign cbFrame cbFrame + 8 %assign frm_saved_r15 -cbFrame push r14 SEH64_PUSH_GREG r14 %assign cbFrame cbFrame + 8 %assign frm_saved_r14 -cbFrame push r13 SEH64_PUSH_GREG r13 %assign cbFrame cbFrame + 8 %assign frm_saved_r13 -cbFrame push r12 SEH64_PUSH_GREG r12 %assign cbFrame cbFrame + 8 %assign frm_saved_r12 -cbFrame push rbx SEH64_PUSH_GREG rbx %assign cbFrame cbFrame + 8 %assign frm_saved_rbx -cbFrame %ifdef ASM_CALL64_MSC push rsi SEH64_PUSH_GREG rsi %assign cbFrame cbFrame + 8 %assign frm_saved_rsi -cbFrame push rdi SEH64_PUSH_GREG rdi %assign cbFrame cbFrame + 8 %assign frm_saved_rdi -cbFrame %endif %endmacro ;; @def POP_CALLEE_PRESERVED_REGISTERS ; Counterpart to PUSH_CALLEE_PRESERVED_REGISTERS for use in the epilogue. %macro POP_CALLEE_PRESERVED_REGISTERS 0 %ifdef ASM_CALL64_MSC pop rdi %assign cbFrame cbFrame - 8 %undef frm_saved_rdi pop rsi %assign cbFrame cbFrame - 8 %undef frm_saved_rsi %endif pop rbx %assign cbFrame cbFrame - 8 %undef frm_saved_rbx pop r12 %assign cbFrame cbFrame - 8 %undef frm_saved_r12 pop r13 %assign cbFrame cbFrame - 8 %undef frm_saved_r13 pop r14 %assign cbFrame cbFrame - 8 %undef frm_saved_r14 pop r15 %assign cbFrame cbFrame - 8 %undef frm_saved_r15 %endmacro ;; @def PUSH_RELEVANT_SEGMENT_REGISTERS ; Macro saving all segment registers on the stack. ; @param 1 Full width register name. ; @param 2 16-bit register name for \a 1. ; @cobbers rax, rdx, rcx %macro PUSH_RELEVANT_SEGMENT_REGISTERS 2 %ifndef VBOX_SKIP_RESTORE_SEG %error untested code. probably does not work any more! %ifndef HM_64_BIT_USE_NULL_SEL mov %2, es push %1 mov %2, ds push %1 %endif ; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode, ; Solaris OTOH doesn't and we must save it. mov ecx, MSR_K8_FS_BASE rdmsr push rdx push rax %ifndef HM_64_BIT_USE_NULL_SEL push fs %endif ; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel. ; The same happens on exit. mov ecx, MSR_K8_GS_BASE rdmsr push rdx push rax %ifndef HM_64_BIT_USE_NULL_SEL push gs %endif %endif ; !VBOX_SKIP_RESTORE_SEG %endmacro ; PUSH_RELEVANT_SEGMENT_REGISTERS ;; @def POP_RELEVANT_SEGMENT_REGISTERS ; Macro restoring all segment registers on the stack. ; @param 1 Full width register name. ; @param 2 16-bit register name for \a 1. ; @cobbers rax, rdx, rcx %macro POP_RELEVANT_SEGMENT_REGISTERS 2 %ifndef VBOX_SKIP_RESTORE_SEG %error untested code. probably does not work any more! ; Note: do not step through this code with a debugger! %ifndef HM_64_BIT_USE_NULL_SEL xor eax, eax mov ds, ax mov es, ax mov fs, ax mov gs, ax %endif %ifndef HM_64_BIT_USE_NULL_SEL pop gs %endif pop rax pop rdx mov ecx, MSR_K8_GS_BASE wrmsr %ifndef HM_64_BIT_USE_NULL_SEL pop fs %endif pop rax pop rdx mov ecx, MSR_K8_FS_BASE wrmsr ; Now it's safe to step again %ifndef HM_64_BIT_USE_NULL_SEL pop %1 mov ds, %2 pop %1 mov es, %2 %endif %endif ; !VBOX_SKIP_RESTORE_SEG %endmacro ; POP_RELEVANT_SEGMENT_REGISTERS ;********************************************************************************************************************************* ;* External Symbols * ;********************************************************************************************************************************* %ifdef VBOX_WITH_KERNEL_USING_XMM extern NAME(CPUMIsGuestFPUStateActive) %endif BEGINCODE ;; ; Used on platforms with poor inline assembly support to retrieve all the ; info from the CPU and put it in the @a pRestoreHost structure. ; ; @returns VBox status code ; @param pRestoreHost msc: rcx gcc: rdi Pointer to the RestoreHost struct. ; @param fHaveFsGsBase msc: dl gcc: sil Whether we can use rdfsbase or not. ; ALIGNCODE(64) BEGINPROC hmR0VmxExportHostSegmentRegsAsmHlp %ifdef ASM_CALL64_MSC %define pRestoreHost rcx %elifdef ASM_CALL64_GCC %define pRestoreHost rdi %else %error Unknown calling convension. %endif SEH64_END_PROLOGUE ; Start with the FS and GS base so we can trash DL/SIL. %ifdef ASM_CALL64_MSC or dl, dl %else or sil, sil %endif jz .use_rdmsr_for_fs_and_gs_base rdfsbase rax mov [pRestoreHost + VMXRESTOREHOST.uHostFSBase], rax rdgsbase rax mov [pRestoreHost + VMXRESTOREHOST.uHostGSBase], rax .done_fs_and_gs_base: ; TR, GDTR and IDTR str [pRestoreHost + VMXRESTOREHOST.uHostSelTR] sgdt [pRestoreHost + VMXRESTOREHOST.HostGdtr] sidt [pRestoreHost + VMXRESTOREHOST.HostIdtr] ; Segment registers. xor eax, eax mov eax, cs mov [pRestoreHost + VMXRESTOREHOST.uHostSelCS], ax mov eax, ss mov [pRestoreHost + VMXRESTOREHOST.uHostSelSS], ax mov eax, gs mov [pRestoreHost + VMXRESTOREHOST.uHostSelGS], ax mov eax, fs mov [pRestoreHost + VMXRESTOREHOST.uHostSelFS], ax mov eax, es mov [pRestoreHost + VMXRESTOREHOST.uHostSelES], ax mov eax, ds mov [pRestoreHost + VMXRESTOREHOST.uHostSelDS], ax ret ALIGNCODE(16) .use_rdmsr_for_fs_and_gs_base: %ifdef ASM_CALL64_MSC mov r8, pRestoreHost %endif mov ecx, MSR_K8_FS_BASE rdmsr shl rdx, 32 or rdx, rax mov [r8 + VMXRESTOREHOST.uHostFSBase], rdx mov ecx, MSR_K8_GS_BASE rdmsr shl rdx, 32 or rdx, rax mov [r8 + VMXRESTOREHOST.uHostGSBase], rdx %ifdef ASM_CALL64_MSC mov pRestoreHost, r8 %endif jmp .done_fs_and_gs_base %undef pRestoreHost ENDPROC hmR0VmxExportHostSegmentRegsAsmHlp ;; ; Restores host-state fields. ; ; @returns VBox status code ; @param f32RestoreHost msc: ecx gcc: edi RestoreHost flags. ; @param pRestoreHost msc: rdx gcc: rsi Pointer to the RestoreHost struct. ; ALIGNCODE(64) BEGINPROC VMXRestoreHostState %ifndef ASM_CALL64_GCC ; Use GCC's input registers since we'll be needing both rcx and rdx further ; down with the wrmsr instruction. Use the R10 and R11 register for saving ; RDI and RSI since MSC preserve the two latter registers. mov r10, rdi mov r11, rsi mov rdi, rcx mov rsi, rdx %endif SEH64_END_PROLOGUE .restore_gdtr: test edi, VMX_RESTORE_HOST_GDTR jz .restore_idtr lgdt [rsi + VMXRESTOREHOST.HostGdtr] .restore_idtr: test edi, VMX_RESTORE_HOST_IDTR jz .restore_ds lidt [rsi + VMXRESTOREHOST.HostIdtr] .restore_ds: test edi, VMX_RESTORE_HOST_SEL_DS jz .restore_es mov ax, [rsi + VMXRESTOREHOST.uHostSelDS] mov ds, eax .restore_es: test edi, VMX_RESTORE_HOST_SEL_ES jz .restore_tr mov ax, [rsi + VMXRESTOREHOST.uHostSelES] mov es, eax .restore_tr: test edi, VMX_RESTORE_HOST_SEL_TR jz .restore_fs ; When restoring the TR, we must first clear the busy flag or we'll end up faulting. mov dx, [rsi + VMXRESTOREHOST.uHostSelTR] mov ax, dx and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset test edi, VMX_RESTORE_HOST_GDT_READ_ONLY | VMX_RESTORE_HOST_GDT_NEED_WRITABLE jnz .gdt_readonly_or_need_writable add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt. and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit) ltr dx .restore_fs: ; ; When restoring the selector values for FS and GS, we'll temporarily trash ; the base address (at least the high 32-bit bits, but quite possibly the ; whole base address), the wrmsr will restore it correctly. (VT-x actually ; restores the base correctly when leaving guest mode, but not the selector ; value, so there is little problem with interrupts being enabled prior to ; this restore job.) ; We'll disable ints once for both FS and GS as that's probably faster. ; test edi, VMX_RESTORE_HOST_SEL_FS | VMX_RESTORE_HOST_SEL_GS jz .restore_success pushfq cli ; (see above) test edi, VMX_RESTORE_HOST_CAN_USE_WRFSBASE_AND_WRGSBASE jz .restore_fs_using_wrmsr .restore_fs_using_wrfsbase: test edi, VMX_RESTORE_HOST_SEL_FS jz .restore_gs_using_wrgsbase mov rax, qword [rsi + VMXRESTOREHOST.uHostFSBase] mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS] mov fs, ecx wrfsbase rax .restore_gs_using_wrgsbase: test edi, VMX_RESTORE_HOST_SEL_GS jz .restore_flags mov rax, qword [rsi + VMXRESTOREHOST.uHostGSBase] mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS] mov gs, ecx wrgsbase rax .restore_flags: popfq .restore_success: mov eax, VINF_SUCCESS %ifndef ASM_CALL64_GCC ; Restore RDI and RSI on MSC. mov rdi, r10 mov rsi, r11 %endif ret ALIGNCODE(8) .gdt_readonly_or_need_writable: test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE jnz .gdt_readonly_need_writable .gdt_readonly: mov rcx, cr0 mov r9, rcx add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt. and rcx, ~X86_CR0_WP mov cr0, rcx and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit) ltr dx mov cr0, r9 jmp .restore_fs ALIGNCODE(8) .gdt_readonly_need_writable: add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit) lgdt [rsi + VMXRESTOREHOST.HostGdtrRw] ltr dx lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; load the original GDT jmp .restore_fs ALIGNCODE(8) .restore_fs_using_wrmsr: test edi, VMX_RESTORE_HOST_SEL_FS jz .restore_gs_using_wrmsr mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi mov cx, word [rsi + VMXRESTOREHOST.uHostSelFS] mov fs, ecx mov ecx, MSR_K8_FS_BASE wrmsr .restore_gs_using_wrmsr: test edi, VMX_RESTORE_HOST_SEL_GS jz .restore_flags mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi mov cx, word [rsi + VMXRESTOREHOST.uHostSelGS] mov gs, ecx mov ecx, MSR_K8_GS_BASE wrmsr jmp .restore_flags ENDPROC VMXRestoreHostState ;; ; Clears the MDS buffers using VERW. ALIGNCODE(16) BEGINPROC hmR0MdsClear SEH64_END_PROLOGUE sub xSP, xCB mov [xSP], ds verw [xSP] add xSP, xCB ret ENDPROC hmR0MdsClear ;; ; Dispatches an NMI to the host. ; ALIGNCODE(16) BEGINPROC VMXDispatchHostNmi ; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts". SEH64_END_PROLOGUE int 2 ret ENDPROC VMXDispatchHostNmi ;; ; Common restore logic for success and error paths. We duplicate this because we ; don't want to waste writing the VINF_SUCCESS return value to the stack in the ; regular code path. ; ; @param 1 Zero if regular return, non-zero if error return. Controls label emission. ; @param 2 fLoadSaveGuestXcr0 value ; @param 3 The (HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY) + HM_WSF_IBPB_EXIT value. ; The entry values are either all set or not at all, as we're too lazy to flesh out all the variants. ; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor. ; ; @note Important that this does not modify cbFrame or rsp. %macro RESTORE_STATE_VMX 4 ; Restore base and limit of the IDTR & GDTR. %ifndef VMX_SKIP_IDTR lidt [rsp + cbFrame + frm_saved_idtr] %endif %ifndef VMX_SKIP_GDTR lgdt [rsp + cbFrame + frm_saved_gdtr] %endif ; Save the guest state and restore the non-volatile registers. We use rcx=pGstCtx (&pVCpu->cpum.GstCtx) here. mov [rsp + cbFrame + frm_guest_rcx], rcx mov rcx, [rsp + cbFrame + frm_pGstCtx] mov qword [rcx + CPUMCTX.eax], rax mov qword [rcx + CPUMCTX.edx], rdx rdtsc mov qword [rcx + CPUMCTX.ebp], rbp lea rbp, [rsp + cbFrame] ; re-establish the frame pointer as early as possible. shl rdx, 20h or rax, rdx ; TSC value in RAX mov rdx, [rbp + frm_guest_rcx] mov qword [rcx + CPUMCTX.ecx], rdx mov rdx, SPECTRE_FILLER ; FILLER in RDX mov qword [rcx + GVMCPU.hmr0 + HMR0PERVCPU.uTscExit - VMCPU.cpum.GstCtx], rax mov qword [rcx + CPUMCTX.r8], r8 mov r8, rdx mov qword [rcx + CPUMCTX.r9], r9 mov r9, rdx mov qword [rcx + CPUMCTX.r10], r10 mov r10, rdx mov qword [rcx + CPUMCTX.r11], r11 mov r11, rdx mov qword [rcx + CPUMCTX.esi], rsi %ifdef ASM_CALL64_MSC mov rsi, [rbp + frm_saved_rsi] %else mov rsi, rdx %endif mov qword [rcx + CPUMCTX.edi], rdi %ifdef ASM_CALL64_MSC mov rdi, [rbp + frm_saved_rdi] %else mov rdi, rdx %endif mov qword [rcx + CPUMCTX.ebx], rbx mov rbx, [rbp + frm_saved_rbx] mov qword [rcx + CPUMCTX.r12], r12 mov r12, [rbp + frm_saved_r12] mov qword [rcx + CPUMCTX.r13], r13 mov r13, [rbp + frm_saved_r13] mov qword [rcx + CPUMCTX.r14], r14 mov r14, [rbp + frm_saved_r14] mov qword [rcx + CPUMCTX.r15], r15 mov r15, [rbp + frm_saved_r15] mov rax, cr2 mov qword [rcx + CPUMCTX.cr2], rax mov rax, rdx %if %4 != 0 ; Save the context pointer in r8 for the SSE save/restore. mov r8, rcx %endif %if %3 & HM_WSF_IBPB_EXIT ; Fight spectre (trashes rax, rdx and rcx). %if %1 = 0 ; Skip this in failure branch (=> guru) mov ecx, MSR_IA32_PRED_CMD mov eax, MSR_IA32_PRED_CMD_F_IBPB xor edx, edx wrmsr %endif %endif %ifndef VMX_SKIP_TR ; Restore TSS selector; must mark it as not busy before using ltr! ; ASSUME that this is supposed to be 'BUSY' (saves 20-30 ticks on the T42p). %ifndef VMX_SKIP_GDTR lgdt [rbp + frm_saved_gdtr] %endif movzx eax, word [rbp + frm_saved_tr] mov ecx, eax and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset add rax, [rbp + frm_saved_gdtr + 2] ; eax <- GDTR.address + descriptor offset and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit) ltr cx %endif movzx edx, word [rbp + frm_saved_ldtr] test edx, edx jz %%skip_ldt_write lldt dx %%skip_ldt_write: %if %1 != 0 .return_after_vmwrite_error: %endif ; Restore segment registers. ;POP_RELEVANT_SEGMENT_REGISTERS rax, ax - currently broken. %if %2 != 0 ; Restore the host XCR0. xor ecx, ecx mov eax, [rbp + frm_uHostXcr0] mov edx, [rbp + frm_uHostXcr0 + 4] xsetbv %endif %endmacro ; RESTORE_STATE_VMX ;; ; hmR0VmxStartVm template ; ; @param 1 The suffix of the variation. ; @param 2 fLoadSaveGuestXcr0 value ; @param 3 The HM_WSF_IBPB_ENTRY + HM_WSF_IBPB_EXIT value. ; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor. ; Drivers shouldn't use AVX registers without saving+loading: ; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396 ; However the compiler docs have different idea: ; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx ; We'll go with the former for now. ; %macro hmR0VmxStartVmTemplate 4 ;; ; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode) ; ; @returns VBox status code ; @param pVmcsInfo msc:rcx, gcc:rdi Pointer to the VMCS info (for cached host RIP and RSP). ; @param pVCpu msc:rdx, gcc:rsi The cross context virtual CPU structure of the calling EMT. ; @param fResume msc:r8l, gcc:dl Whether to use vmlauch/vmresume. ; ALIGNCODE(64) BEGINPROC RT_CONCAT(hmR0VmxStartVm,%1) %ifdef VBOX_WITH_KERNEL_USING_XMM %if %4 = 0 ; ; The non-saving variant will currently check the two SSE preconditions and pick ; the right variant to continue with. Later we can see if we can't manage to ; move these decisions into hmR0VmxUpdateStartVmFunction(). ; %ifdef ASM_CALL64_MSC test byte [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1 %else test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1 %endif jz .save_xmm_no_need %ifdef ASM_CALL64_MSC cmp dword [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0 %else cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0 %endif je RT_CONCAT3(hmR0VmxStartVm,%1,_SseManual) jmp RT_CONCAT3(hmR0VmxStartVm,%1,_SseXSave) .save_xmm_no_need: %endif %endif push xBP SEH64_PUSH_xBP mov xBP, xSP SEH64_SET_FRAME_xBP 0 pushf cli %define frm_fRFlags -008h %define frm_pGstCtx -010h ; Where we stash guest CPU context for use after the vmrun. %define frm_uHostXcr0 -020h ; 128-bit %define frm_saved_gdtr -02ah ; 16+64: Only used when VMX_SKIP_GDTR isn't defined %define frm_saved_tr -02ch ; 16-bit: Only used when VMX_SKIP_TR isn't defined %define frm_MDS_seg -030h ; 16-bit: Temporary storage for the MDS flushing. %define frm_saved_idtr -03ah ; 16+64: Only used when VMX_SKIP_IDTR isn't defined %define frm_saved_ldtr -03ch ; 16-bit: always saved. %define frm_rcError -040h ; 32-bit: Error status code (not used in the success path) %define frm_guest_rcx -048h ; Temporary storage slot for guest RCX. %if %4 = 0 %assign cbFrame 048h %else %define frm_saved_xmm6 -050h %define frm_saved_xmm7 -060h %define frm_saved_xmm8 -070h %define frm_saved_xmm9 -080h %define frm_saved_xmm10 -090h %define frm_saved_xmm11 -0a0h %define frm_saved_xmm12 -0b0h %define frm_saved_xmm13 -0c0h %define frm_saved_xmm14 -0d0h %define frm_saved_xmm15 -0e0h %define frm_saved_mxcsr -0f0h %assign cbFrame 0f0h %endif %assign cbBaseFrame cbFrame sub rsp, cbFrame - 8h SEH64_ALLOCATE_STACK cbFrame ; Save all general purpose host registers. PUSH_CALLEE_PRESERVED_REGISTERS ;PUSH_RELEVANT_SEGMENT_REGISTERS xAX, ax - currently broken SEH64_END_PROLOGUE ; ; Unify the input parameter registers: r9=pVmcsInfo, rsi=pVCpu, bl=fResume, rdi=&pVCpu->cpum.GstCtx; ; %ifdef ASM_CALL64_GCC mov r9, rdi ; pVmcsInfo mov ebx, edx ; fResume %else mov r9, rcx ; pVmcsInfo mov rsi, rdx ; pVCpu mov ebx, r8d ; fResume %endif lea rdi, [rsi + VMCPU.cpum.GstCtx] mov [rbp + frm_pGstCtx], rdi %ifdef VBOX_STRICT ; ; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change. ; cmp byte [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fLoadSaveGuestXcr0], %2 mov eax, VERR_VMX_STARTVM_PRECOND_0 jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return) mov eax, [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fWorldSwitcher] and eax, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT cmp eax, %3 mov eax, VERR_VMX_STARTVM_PRECOND_1 jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return) %ifdef VBOX_WITH_KERNEL_USING_XMM mov eax, VERR_VMX_STARTVM_PRECOND_2 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1 %if %4 = 0 jnz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return) %else jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return) mov eax, VERR_VMX_STARTVM_PRECOND_3 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0 %if %4 = 1 jne NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return) %elif %4 = 2 je NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).precond_failure_return) %else %error Invalid template parameter 4. %endif %endif %endif %endif ; VBOX_STRICT %if %4 != 0 ; Save the non-volatile SSE host register state. movdqa [rbp + frm_saved_xmm6 ], xmm6 movdqa [rbp + frm_saved_xmm7 ], xmm7 movdqa [rbp + frm_saved_xmm8 ], xmm8 movdqa [rbp + frm_saved_xmm9 ], xmm9 movdqa [rbp + frm_saved_xmm10], xmm10 movdqa [rbp + frm_saved_xmm11], xmm11 movdqa [rbp + frm_saved_xmm12], xmm12 movdqa [rbp + frm_saved_xmm13], xmm13 movdqa [rbp + frm_saved_xmm14], xmm14 movdqa [rbp + frm_saved_xmm15], xmm15 stmxcsr [rbp + frm_saved_mxcsr] ; Load the guest state related to the above non-volatile and volatile SSE registers. Trashes rcx, eax and edx. lea rcx, [rdi + CPUMCTX.XState] %if %4 = 1 ; manual movdqa xmm0, [rcx + XMM_OFF_IN_X86FXSTATE + 000h] movdqa xmm1, [rcx + XMM_OFF_IN_X86FXSTATE + 010h] movdqa xmm2, [rcx + XMM_OFF_IN_X86FXSTATE + 020h] movdqa xmm3, [rcx + XMM_OFF_IN_X86FXSTATE + 030h] movdqa xmm4, [rcx + XMM_OFF_IN_X86FXSTATE + 040h] movdqa xmm5, [rcx + XMM_OFF_IN_X86FXSTATE + 050h] movdqa xmm6, [rcx + XMM_OFF_IN_X86FXSTATE + 060h] movdqa xmm7, [rcx + XMM_OFF_IN_X86FXSTATE + 070h] movdqa xmm8, [rcx + XMM_OFF_IN_X86FXSTATE + 080h] movdqa xmm9, [rcx + XMM_OFF_IN_X86FXSTATE + 090h] movdqa xmm10, [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h] movdqa xmm11, [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h] movdqa xmm12, [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h] movdqa xmm13, [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h] movdqa xmm14, [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h] movdqa xmm15, [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h] ldmxcsr [rcx + X86FXSTATE.MXCSR] %elif %4 = 2 ; use xrstor/xsave mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask] and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS xor edx, edx xrstor [rcx] %else %error invalid template parameter 4 %endif %endif %if %2 != 0 ; Save the host XCR0 and load the guest one if necessary. ; Note! Trashes rax, rdx and rcx. xor ecx, ecx xgetbv ; save the host one on the stack mov [rbp + frm_uHostXcr0], eax mov [rbp + frm_uHostXcr0 + 4], edx mov eax, [rdi + CPUMCTX.aXcr] ; load the guest one mov edx, [rdi + CPUMCTX.aXcr + 4] xor ecx, ecx ; paranoia; indicate that we must restore XCR0 (popped into ecx, thus 0) xsetbv %endif ; Save host LDTR. sldt word [rbp + frm_saved_ldtr] %ifndef VMX_SKIP_TR ; The host TR limit is reset to 0x67; save & restore it manually. str word [rbp + frm_saved_tr] %endif %ifndef VMX_SKIP_GDTR ; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly! sgdt [rbp + frm_saved_gdtr] %endif %ifndef VMX_SKIP_IDTR sidt [rbp + frm_saved_idtr] %endif ; Load CR2 if necessary (expensive as writing CR2 is a synchronizing instruction - (bird: still expensive on 10980xe)). mov rcx, qword [rdi + CPUMCTX.cr2] mov rdx, cr2 cmp rcx, rdx je .skip_cr2_write mov cr2, rcx .skip_cr2_write: ; Set the vmlaunch/vmresume "return" host RIP and RSP values if they've changed (unlikly). ; The vmwrite isn't quite for free (on an 10980xe at least), thus we check if anything changed ; before writing here. lea rcx, [NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) wrt rip] cmp rcx, [r9 + VMXVMCSINFO.uHostRip] jne .write_host_rip .wrote_host_rip: cmp rsp, [r9 + VMXVMCSINFO.uHostRsp] jne .write_host_rsp .wrote_host_rsp: ; ; Fight spectre and similar. Trashes rax, rcx, and rdx. ; %if %3 & (HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY) ; The eax:edx value is the same for the first two. AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D) mov eax, MSR_IA32_PRED_CMD_F_IBPB xor edx, edx %endif %if %3 & HM_WSF_IBPB_ENTRY ; Indirect branch barrier. mov ecx, MSR_IA32_PRED_CMD wrmsr %endif %if %3 & HM_WSF_L1D_ENTRY ; Level 1 data cache flush. mov ecx, MSR_IA32_FLUSH_CMD wrmsr %elif %3 & HM_WSF_MDS_ENTRY ; MDS flushing is included in L1D_FLUSH mov word [rbp + frm_MDS_seg], ds verw word [rbp + frm_MDS_seg] %endif ; Resume or start VM? cmp bl, 0 ; fResume ; Load guest general purpose registers. mov rax, qword [rdi + CPUMCTX.eax] mov rbx, qword [rdi + CPUMCTX.ebx] mov rcx, qword [rdi + CPUMCTX.ecx] mov rdx, qword [rdi + CPUMCTX.edx] mov rbp, qword [rdi + CPUMCTX.ebp] mov rsi, qword [rdi + CPUMCTX.esi] mov r8, qword [rdi + CPUMCTX.r8] mov r9, qword [rdi + CPUMCTX.r9] mov r10, qword [rdi + CPUMCTX.r10] mov r11, qword [rdi + CPUMCTX.r11] mov r12, qword [rdi + CPUMCTX.r12] mov r13, qword [rdi + CPUMCTX.r13] mov r14, qword [rdi + CPUMCTX.r14] mov r15, qword [rdi + CPUMCTX.r15] mov rdi, qword [rdi + CPUMCTX.edi] je .vmlaunch64_launch vmresume jc NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_invalid_vmcs_ptr) jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_start_failed) jmp NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) ; here if vmresume detected a failure .vmlaunch64_launch: vmlaunch jc NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_invalid_vmcs_ptr) jz NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmxstart64_start_failed) jmp NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) ; here if vmlaunch detected a failure ; Put these two outside the normal code path as they should rarely change. ALIGNCODE(8) .write_host_rip: %ifdef VBOX_WITH_STATISTICS inc qword [rsi + VMCPU.hm + HMCPU.StatVmxWriteHostRip] %endif mov [r9 + VMXVMCSINFO.uHostRip], rcx mov eax, VMX_VMCS_HOST_RIP ;; @todo It is only strictly necessary to write VMX_VMCS_HOST_RIP when vmwrite rax, rcx ;; the VMXVMCSINFO::pfnStartVM function changes (eventually %ifdef VBOX_STRICT ;; take the Windows/SSE stuff into account then)... jna NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmwrite_failed) %endif jmp .wrote_host_rip ALIGNCODE(8) .write_host_rsp: %ifdef VBOX_WITH_STATISTICS inc qword [rsi + VMCPU.hm + HMCPU.StatVmxWriteHostRsp] %endif mov [r9 + VMXVMCSINFO.uHostRsp], rsp mov eax, VMX_VMCS_HOST_RSP vmwrite rax, rsp %ifdef VBOX_STRICT jna NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1).vmwrite_failed) %endif jmp .wrote_host_rsp ALIGNCODE(64) GLOBALNAME RT_CONCAT(hmR0VmxStartVmHostRIP,%1) RESTORE_STATE_VMX 0, %2, %3, %4 mov eax, VINF_SUCCESS .vmstart64_end: %if %4 != 0 mov r11d, eax ; save the return code. ; Save the guest SSE state related to non-volatile and volatile SSE registers. lea rcx, [r8 + CPUMCTX.XState] %if %4 = 1 ; manual stmxcsr [rcx + X86FXSTATE.MXCSR] movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 000h], xmm0 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 010h], xmm1 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 020h], xmm2 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 030h], xmm3 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 040h], xmm4 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 050h], xmm5 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 060h], xmm6 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 070h], xmm7 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 080h], xmm8 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 090h], xmm9 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15 %elif %4 = 2 ; use xrstor/xsave mov eax, [r8 + CPUMCTX.fXStateMask] and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS xor edx, edx xsave [rcx] %else %error invalid template parameter 4 %endif ; Restore the host non-volatile SSE register state. ldmxcsr [rbp + frm_saved_mxcsr] movdqa xmm6, [rbp + frm_saved_xmm6 ] movdqa xmm7, [rbp + frm_saved_xmm7 ] movdqa xmm8, [rbp + frm_saved_xmm8 ] movdqa xmm9, [rbp + frm_saved_xmm9 ] movdqa xmm10, [rbp + frm_saved_xmm10] movdqa xmm11, [rbp + frm_saved_xmm11] movdqa xmm12, [rbp + frm_saved_xmm12] movdqa xmm13, [rbp + frm_saved_xmm13] movdqa xmm14, [rbp + frm_saved_xmm14] movdqa xmm15, [rbp + frm_saved_xmm15] mov eax, r11d %endif ; %4 != 0 lea rsp, [rbp + frm_fRFlags] popf leave ret ; ; Error returns. ; %ifdef VBOX_STRICT .vmwrite_failed: mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_FIELD jz .return_after_vmwrite_error mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_PTR jmp .return_after_vmwrite_error %endif .vmxstart64_invalid_vmcs_ptr: mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_INVALID_VMCS_PTR_TO_START_VM jmp .vmstart64_error_return .vmxstart64_start_failed: mov dword [rsp + cbFrame + frm_rcError], VERR_VMX_UNABLE_TO_START_VM .vmstart64_error_return: RESTORE_STATE_VMX 1, %2, %3, %4 mov eax, [rbp + frm_rcError] jmp .vmstart64_end %ifdef VBOX_STRICT ; Precondition checks failed. .precond_failure_return: POP_CALLEE_PRESERVED_REGISTERS %if cbFrame != cbBaseFrame %error Bad frame size value: cbFrame, expected cbBaseFrame %endif jmp .vmstart64_end %endif %undef frm_fRFlags %undef frm_pGstCtx %undef frm_uHostXcr0 %undef frm_saved_gdtr %undef frm_saved_tr %undef frm_fNoRestoreXcr0 %undef frm_saved_idtr %undef frm_saved_ldtr %undef frm_rcError %undef frm_guest_rax %undef cbFrame ENDPROC RT_CONCAT(hmR0VmxStartVm,%1) %ifdef ASM_FORMAT_ELF size NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) NAME(RT_CONCAT(hmR0VmxStartVm,%1) %+ _EndProc) - NAME(RT_CONCAT(hmR0VmxStartVmHostRIP,%1)) %endif %endmacro ; hmR0VmxStartVmTemplate %macro hmR0VmxStartVmSseTemplate 2 hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, 0 | 0 | 0 | 0 , %1 hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, 0 | 0 | 0 | 0 , %1 hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | 0 | 0 , %1 hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | 0 | 0 , %1 hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | 0 | 0 , %1 hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | 0 | 0 , %1 hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | 0 , %1 hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | 0 , %1 hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, 0 | 0 | HM_WSF_MDS_ENTRY | 0 , %1 hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, 0 | 0 | HM_WSF_MDS_ENTRY | 0 , %1 hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | 0 , %1 hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | 0 , %1 hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1 hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1 hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1 hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_SansIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | 0 , %1 hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, 0 | 0 | 0 | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, 0 | 0 | 0 | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | 0 | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | 0 | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_SansMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | 0 | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, 0 | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, 0 | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_SansL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | 0 | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _SansXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _WithXcr0_SansIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, 0 | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _SansXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 0, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1 hmR0VmxStartVmTemplate _WithXcr0_WithIbpbEntry_WithL1dEntry_WithMdsEntry_WithIbpbExit %+ %2, 1, HM_WSF_IBPB_ENTRY | HM_WSF_L1D_ENTRY | HM_WSF_MDS_ENTRY | HM_WSF_IBPB_EXIT, %1 %endmacro hmR0VmxStartVmSseTemplate 0,, %ifdef VBOX_WITH_KERNEL_USING_XMM hmR0VmxStartVmSseTemplate 1,_SseManual hmR0VmxStartVmSseTemplate 2,_SseXSave %endif ;; ; hmR0SvmVmRun template ; ; @param 1 The suffix of the variation. ; @param 2 fLoadSaveGuestXcr0 value ; @param 3 The HM_WSF_IBPB_ENTRY + HM_WSF_IBPB_EXIT value. ; @param 4 The SSE saving/restoring: 0 to do nothing, 1 to do it manually, 2 to use xsave/xrstor. ; Drivers shouldn't use AVX registers without saving+loading: ; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396 ; However the compiler docs have different idea: ; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx ; We'll go with the former for now. ; %macro hmR0SvmVmRunTemplate 4 ;; ; Prepares for and executes VMRUN (32-bit and 64-bit guests). ; ; @returns VBox status code ; @param pVM msc:rcx,gcc:rdi The cross context VM structure (unused). ; @param pVCpu msc:rdx,gcc:rsi The cross context virtual CPU structure of the calling EMT. ; @param HCPhysVmcb msc:r8, gcc:rdx Physical address of guest VMCB. ; ALIGNCODE(64) ; This + immediate optimizations causes serious trouble for yasm and the SEH frames: prologue -28 bytes, must be <256 ; So the SEH64_XXX stuff is currently not operational. BEGINPROC RT_CONCAT(hmR0SvmVmRun,%1) %ifdef VBOX_WITH_KERNEL_USING_XMM %if %4 = 0 ; ; The non-saving variant will currently check the two SSE preconditions and pick ; the right variant to continue with. Later we can see if we can't manage to ; move these decisions into hmR0SvmUpdateVmRunFunction(). ; %ifdef ASM_CALL64_MSC test byte [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1 %else test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1 %endif jz .save_xmm_no_need %ifdef ASM_CALL64_MSC cmp dword [rdx + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0 %else cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0 %endif je RT_CONCAT3(hmR0SvmVmRun,%1,_SseManual) jmp RT_CONCAT3(hmR0SvmVmRun,%1,_SseXSave) .save_xmm_no_need: %endif %endif push rbp SEH64_PUSH_xBP mov rbp, rsp SEH64_SET_FRAME_xBP 0 pushf %assign cbFrame 30h %if %4 != 0 %assign cbFrame cbFrame + 16 * 11 ; Reserve space for 10x 128-bit XMM registers and MXCSR (32-bit) %endif %assign cbBaseFrame cbFrame sub rsp, cbFrame - 8h ; We subtract 8 bytes for the above pushf SEH64_ALLOCATE_STACK cbFrame ; And we have CALLEE_PRESERVED_REGISTER_COUNT following it. %define frm_fRFlags -008h %define frm_uHostXcr0 -018h ; 128-bit ;%define frm_fNoRestoreXcr0 -020h ; Non-zero if we should skip XCR0 restoring. %define frm_pGstCtx -028h ; Where we stash guest CPU context for use after the vmrun. %define frm_HCPhysVmcbHost -030h ; Where we stash HCPhysVmcbHost for the vmload after vmrun. %if %4 != 0 %define frm_saved_xmm6 -040h %define frm_saved_xmm7 -050h %define frm_saved_xmm8 -060h %define frm_saved_xmm9 -070h %define frm_saved_xmm10 -080h %define frm_saved_xmm11 -090h %define frm_saved_xmm12 -0a0h %define frm_saved_xmm13 -0b0h %define frm_saved_xmm14 -0c0h %define frm_saved_xmm15 -0d0h %define frm_saved_mxcsr -0e0h %endif ; Manual save and restore: ; - General purpose registers except RIP, RSP, RAX ; ; Trashed: ; - CR2 (we don't care) ; - LDTR (reset to 0) ; - DRx (presumably not changed at all) ; - DR7 (reset to 0x400) ; Save all general purpose host registers. PUSH_CALLEE_PRESERVED_REGISTERS SEH64_END_PROLOGUE %if cbFrame != (cbBaseFrame + 8 * CALLEE_PRESERVED_REGISTER_COUNT) %error Bad cbFrame value %endif ; Shuffle parameter registers so that r8=HCPhysVmcb and rsi=pVCpu. (rdx & rcx will soon be trashed.) %ifdef ASM_CALL64_GCC mov r8, rdx ; Put HCPhysVmcb in r8 like on MSC as rdx is trashed below. %else mov rsi, rdx ; Put pVCpu in rsi like on GCC as rdx is trashed below. ;mov rdi, rcx ; Put pVM in rdi like on GCC as rcx is trashed below. %endif %ifdef VBOX_STRICT ; ; Verify template preconditions / parameters to ensure HMSVM.cpp didn't miss some state change. ; cmp byte [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fLoadSaveGuestXcr0], %2 mov eax, VERR_SVM_VMRUN_PRECOND_0 jne .failure_return mov eax, [rsi + GVMCPU.hmr0 + HMR0PERVCPU.fWorldSwitcher] and eax, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT cmp eax, %3 mov eax, VERR_SVM_VMRUN_PRECOND_1 jne .failure_return %ifdef VBOX_WITH_KERNEL_USING_XMM mov eax, VERR_SVM_VMRUN_PRECOND_2 test byte [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fUsedFpuGuest], 1 %if %4 = 0 jnz .failure_return %else jz .failure_return mov eax, VERR_SVM_VMRUN_PRECOND_3 cmp dword [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask], 0 %if %4 = 1 jne .failure_return %elif %4 = 2 je .failure_return %else %error Invalid template parameter 4. %endif %endif %endif %endif ; VBOX_STRICT %if %4 != 0 ; Save the non-volatile SSE host register state. movdqa [rbp + frm_saved_xmm6 ], xmm6 movdqa [rbp + frm_saved_xmm7 ], xmm7 movdqa [rbp + frm_saved_xmm8 ], xmm8 movdqa [rbp + frm_saved_xmm9 ], xmm9 movdqa [rbp + frm_saved_xmm10], xmm10 movdqa [rbp + frm_saved_xmm11], xmm11 movdqa [rbp + frm_saved_xmm12], xmm12 movdqa [rbp + frm_saved_xmm13], xmm13 movdqa [rbp + frm_saved_xmm14], xmm14 movdqa [rbp + frm_saved_xmm15], xmm15 stmxcsr [rbp + frm_saved_mxcsr] ; Load the guest state related to the above non-volatile and volatile SSE registers. Trashes rcx, eax and edx. lea rcx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.XState] %if %4 = 1 ; manual movdqa xmm0, [rcx + XMM_OFF_IN_X86FXSTATE + 000h] movdqa xmm1, [rcx + XMM_OFF_IN_X86FXSTATE + 010h] movdqa xmm2, [rcx + XMM_OFF_IN_X86FXSTATE + 020h] movdqa xmm3, [rcx + XMM_OFF_IN_X86FXSTATE + 030h] movdqa xmm4, [rcx + XMM_OFF_IN_X86FXSTATE + 040h] movdqa xmm5, [rcx + XMM_OFF_IN_X86FXSTATE + 050h] movdqa xmm6, [rcx + XMM_OFF_IN_X86FXSTATE + 060h] movdqa xmm7, [rcx + XMM_OFF_IN_X86FXSTATE + 070h] movdqa xmm8, [rcx + XMM_OFF_IN_X86FXSTATE + 080h] movdqa xmm9, [rcx + XMM_OFF_IN_X86FXSTATE + 090h] movdqa xmm10, [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h] movdqa xmm11, [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h] movdqa xmm12, [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h] movdqa xmm13, [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h] movdqa xmm14, [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h] movdqa xmm15, [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h] ldmxcsr [rcx + X86FXSTATE.MXCSR] %elif %4 = 2 ; use xrstor/xsave mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.fXStateMask] and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS xor edx, edx xrstor [rcx] %else %error invalid template parameter 4 %endif %endif %if %2 != 0 ; Save the host XCR0 and load the guest one if necessary. xor ecx, ecx xgetbv ; save the host XCR0 on the stack mov [rbp + frm_uHostXcr0 + 8], rdx mov [rbp + frm_uHostXcr0 ], rax mov eax, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr] ; load the guest XCR0 mov edx, [rsi + VMCPU.cpum.GstCtx + CPUMCTX.aXcr + 4] xor ecx, ecx ; paranoia xsetbv %endif ; Save host fs, gs, sysenter msr etc. mov rax, [rsi + GVMCPU.hmr0 + HMR0PERVCPU.svm + HMR0CPUSVM.HCPhysVmcbHost] mov qword [rbp + frm_HCPhysVmcbHost], rax ; save for the vmload after vmrun lea rsi, [rsi + VMCPU.cpum.GstCtx] mov qword [rbp + frm_pGstCtx], rsi vmsave %if %3 & HM_WSF_IBPB_ENTRY ; Fight spectre (trashes rax, rdx and rcx). mov ecx, MSR_IA32_PRED_CMD mov eax, MSR_IA32_PRED_CMD_F_IBPB xor edx, edx wrmsr %endif ; Setup rax for VMLOAD. mov rax, r8 ; HCPhysVmcb (64 bits physical address; take low dword only) ; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN). mov rbx, qword [rsi + CPUMCTX.ebx] mov rcx, qword [rsi + CPUMCTX.ecx] mov rdx, qword [rsi + CPUMCTX.edx] mov rdi, qword [rsi + CPUMCTX.edi] mov rbp, qword [rsi + CPUMCTX.ebp] mov r8, qword [rsi + CPUMCTX.r8] mov r9, qword [rsi + CPUMCTX.r9] mov r10, qword [rsi + CPUMCTX.r10] mov r11, qword [rsi + CPUMCTX.r11] mov r12, qword [rsi + CPUMCTX.r12] mov r13, qword [rsi + CPUMCTX.r13] mov r14, qword [rsi + CPUMCTX.r14] mov r15, qword [rsi + CPUMCTX.r15] mov rsi, qword [rsi + CPUMCTX.esi] ; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch. clgi sti ; Load guest FS, GS, Sysenter MSRs etc. vmload ; Run the VM. vmrun ; Save guest fs, gs, sysenter msr etc. vmsave ; Load host fs, gs, sysenter msr etc. mov rax, [rsp + cbFrame + frm_HCPhysVmcbHost] ; load HCPhysVmcbHost (rbp is not operational yet, thus rsp) vmload ; Set the global interrupt flag again, but execute cli to make sure IF=0. cli stgi ; Pop pVCpu (saved above) and save the guest GPRs (sans RSP and RAX). mov rax, [rsp + cbFrame + frm_pGstCtx] ; (rbp still not operational) mov qword [rax + CPUMCTX.edx], rdx mov qword [rax + CPUMCTX.ecx], rcx mov rcx, rax rdtsc mov qword [rcx + CPUMCTX.ebp], rbp lea rbp, [rsp + cbFrame] shl rdx, 20h or rax, rdx ; TSC value in RAX mov qword [rcx + CPUMCTX.r8], r8 mov r8, SPECTRE_FILLER ; SPECTRE filler in R8 mov qword [rcx + CPUMCTX.r9], r9 mov r9, r8 mov qword [rcx + CPUMCTX.r10], r10 mov r10, r8 mov qword [rcx + GVMCPU.hmr0 + HMR0PERVCPU.uTscExit - VMCPU.cpum.GstCtx], rax mov qword [rcx + CPUMCTX.r11], r11 mov r11, r8 mov qword [rcx + CPUMCTX.edi], rdi %ifdef ASM_CALL64_MSC mov rdi, [rbp + frm_saved_rdi] %else mov rdi, r8 %endif mov qword [rcx + CPUMCTX.esi], rsi %ifdef ASM_CALL64_MSC mov rsi, [rbp + frm_saved_rsi] %else mov rsi, r8 %endif mov qword [rcx + CPUMCTX.ebx], rbx mov rbx, [rbp + frm_saved_rbx] mov qword [rcx + CPUMCTX.r12], r12 mov r12, [rbp + frm_saved_r12] mov qword [rcx + CPUMCTX.r13], r13 mov r13, [rbp + frm_saved_r13] mov qword [rcx + CPUMCTX.r14], r14 mov r14, [rbp + frm_saved_r14] mov qword [rcx + CPUMCTX.r15], r15 mov r15, [rbp + frm_saved_r15] %if %4 != 0 ; Set r8 = &pVCpu->cpum.GstCtx; for use below when saving and restoring SSE state. mov r8, rcx %endif %if %3 & HM_WSF_IBPB_EXIT ; Fight spectre (trashes rax, rdx and rcx). mov ecx, MSR_IA32_PRED_CMD mov eax, MSR_IA32_PRED_CMD_F_IBPB xor edx, edx wrmsr %endif %if %2 != 0 ; Restore the host xcr0. xor ecx, ecx mov rdx, [rbp + frm_uHostXcr0 + 8] mov rax, [rbp + frm_uHostXcr0] xsetbv %endif %if %4 != 0 ; Save the guest SSE state related to non-volatile and volatile SSE registers. lea rcx, [r8 + CPUMCTX.XState] %if %4 = 1 ; manual stmxcsr [rcx + X86FXSTATE.MXCSR] movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 000h], xmm0 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 010h], xmm1 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 020h], xmm2 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 030h], xmm3 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 040h], xmm4 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 050h], xmm5 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 060h], xmm6 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 070h], xmm7 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 080h], xmm8 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 090h], xmm9 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14 movdqa [rcx + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15 %elif %4 = 2 ; use xrstor/xsave mov eax, [r8 + CPUMCTX.fXStateMask] and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS xor edx, edx xsave [rcx] %else %error invalid template parameter 4 %endif ; Restore the host non-volatile SSE register state. ldmxcsr [rbp + frm_saved_mxcsr] movdqa xmm6, [rbp + frm_saved_xmm6 ] movdqa xmm7, [rbp + frm_saved_xmm7 ] movdqa xmm8, [rbp + frm_saved_xmm8 ] movdqa xmm9, [rbp + frm_saved_xmm9 ] movdqa xmm10, [rbp + frm_saved_xmm10] movdqa xmm11, [rbp + frm_saved_xmm11] movdqa xmm12, [rbp + frm_saved_xmm12] movdqa xmm13, [rbp + frm_saved_xmm13] movdqa xmm14, [rbp + frm_saved_xmm14] movdqa xmm15, [rbp + frm_saved_xmm15] %endif ; %4 != 0 ; Epilogue (assumes we restored volatile registers above when saving the guest GPRs). mov eax, VINF_SUCCESS add rsp, cbFrame - 8h popf leave ret %ifdef VBOX_STRICT ; Precondition checks failed. .failure_return: POP_CALLEE_PRESERVED_REGISTERS %if cbFrame != cbBaseFrame %error Bad frame size value: cbFrame %endif add rsp, cbFrame - 8h popf leave ret %endif %undef frm_uHostXcr0 %undef frm_fNoRestoreXcr0 %undef frm_pVCpu %undef frm_HCPhysVmcbHost %undef cbFrame ENDPROC RT_CONCAT(hmR0SvmVmRun,%1) %endmacro ; hmR0SvmVmRunTemplate ; ; Instantiate the hmR0SvmVmRun various variations. ; hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit, 0, 0, 0 hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit, 1, 0, 0 hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit, 0, HM_WSF_IBPB_ENTRY, 0 hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit, 1, HM_WSF_IBPB_ENTRY, 0 hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit, 0, HM_WSF_IBPB_EXIT, 0 hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit, 1, HM_WSF_IBPB_EXIT, 0 hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit, 0, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 0 hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit, 1, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 0 %ifdef VBOX_WITH_KERNEL_USING_XMM hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 0, 0, 1 hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseManual, 1, 0, 1 hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 0, HM_WSF_IBPB_ENTRY, 1 hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseManual, 1, HM_WSF_IBPB_ENTRY, 1 hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 0, HM_WSF_IBPB_EXIT, 1 hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseManual, 1, HM_WSF_IBPB_EXIT, 1 hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 0, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 1 hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseManual, 1, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 1 hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 0, 0, 2 hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_SansIbpbExit_SseXSave, 1, 0, 2 hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 0, HM_WSF_IBPB_ENTRY, 2 hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_SansIbpbExit_SseXSave, 1, HM_WSF_IBPB_ENTRY, 2 hmR0SvmVmRunTemplate _SansXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 0, HM_WSF_IBPB_EXIT, 2 hmR0SvmVmRunTemplate _WithXcr0_SansIbpbEntry_WithIbpbExit_SseXSave, 1, HM_WSF_IBPB_EXIT, 2 hmR0SvmVmRunTemplate _SansXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 0, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 2 hmR0SvmVmRunTemplate _WithXcr0_WithIbpbEntry_WithIbpbExit_SseXSave, 1, HM_WSF_IBPB_ENTRY | HM_WSF_IBPB_EXIT, 2 %endif