| 1 | /* $Id: IEMAllThrdTables.h 104357 2024-04-17 21:57:03Z vboxsync $ */
|
|---|
| 2 | /** @file
|
|---|
| 3 | * IEM - Instruction Decoding and Threaded Recompilation, Instruction Tables.
|
|---|
| 4 | */
|
|---|
| 5 |
|
|---|
| 6 | /*
|
|---|
| 7 | * Copyright (C) 2011-2023 Oracle and/or its affiliates.
|
|---|
| 8 | *
|
|---|
| 9 | * This file is part of VirtualBox base platform packages, as
|
|---|
| 10 | * available from https://www.virtualbox.org.
|
|---|
| 11 | *
|
|---|
| 12 | * This program is free software; you can redistribute it and/or
|
|---|
| 13 | * modify it under the terms of the GNU General Public License
|
|---|
| 14 | * as published by the Free Software Foundation, in version 3 of the
|
|---|
| 15 | * License.
|
|---|
| 16 | *
|
|---|
| 17 | * This program is distributed in the hope that it will be useful, but
|
|---|
| 18 | * WITHOUT ANY WARRANTY; without even the implied warranty of
|
|---|
| 19 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|---|
| 20 | * General Public License for more details.
|
|---|
| 21 | *
|
|---|
| 22 | * You should have received a copy of the GNU General Public License
|
|---|
| 23 | * along with this program; if not, see <https://www.gnu.org/licenses>.
|
|---|
| 24 | *
|
|---|
| 25 | * SPDX-License-Identifier: GPL-3.0-only
|
|---|
| 26 | */
|
|---|
| 27 |
|
|---|
| 28 | #ifndef VMM_INCLUDED_SRC_VMMAll_IEMAllThrdTables_h
|
|---|
| 29 | #define VMM_INCLUDED_SRC_VMMAll_IEMAllThrdTables_h
|
|---|
| 30 | #ifndef RT_WITHOUT_PRAGMA_ONCE
|
|---|
| 31 | # pragma once
|
|---|
| 32 | #endif
|
|---|
| 33 |
|
|---|
| 34 |
|
|---|
| 35 | /*********************************************************************************************************************************
|
|---|
| 36 | * Header Files *
|
|---|
| 37 | *********************************************************************************************************************************/
|
|---|
| 38 | #ifndef LOG_GROUP /* defined when included by tstIEMCheckMc.cpp */
|
|---|
| 39 | # define LOG_GROUP LOG_GROUP_IEM_RE_THREADED
|
|---|
| 40 | #endif
|
|---|
| 41 | #define IEM_WITH_CODE_TLB_AND_OPCODE_BUF /* A bit hackish, but its all in IEMInline.h. */
|
|---|
| 42 | #define VMCPU_INCL_CPUM_GST_CTX
|
|---|
| 43 | #include <VBox/vmm/iem.h>
|
|---|
| 44 | #include <VBox/vmm/cpum.h>
|
|---|
| 45 | #include <VBox/vmm/apic.h>
|
|---|
| 46 | #include <VBox/vmm/pdm.h>
|
|---|
| 47 | #include <VBox/vmm/pgm.h>
|
|---|
| 48 | #include <VBox/vmm/iom.h>
|
|---|
| 49 | #include <VBox/vmm/em.h>
|
|---|
| 50 | #include <VBox/vmm/hm.h>
|
|---|
| 51 | #include <VBox/vmm/nem.h>
|
|---|
| 52 | #include <VBox/vmm/gim.h>
|
|---|
| 53 | #ifdef VBOX_WITH_NESTED_HWVIRT_SVM
|
|---|
| 54 | # include <VBox/vmm/em.h>
|
|---|
| 55 | # include <VBox/vmm/hm_svm.h>
|
|---|
| 56 | #endif
|
|---|
| 57 | #ifdef VBOX_WITH_NESTED_HWVIRT_VMX
|
|---|
| 58 | # include <VBox/vmm/hmvmxinline.h>
|
|---|
| 59 | #endif
|
|---|
| 60 | #include <VBox/vmm/tm.h>
|
|---|
| 61 | #include <VBox/vmm/dbgf.h>
|
|---|
| 62 | #include <VBox/vmm/dbgftrace.h>
|
|---|
| 63 | #ifndef TST_IEM_CHECK_MC
|
|---|
| 64 | # include "IEMInternal.h"
|
|---|
| 65 | #endif
|
|---|
| 66 | #include <VBox/vmm/vmcc.h>
|
|---|
| 67 | #include <VBox/log.h>
|
|---|
| 68 | #include <VBox/err.h>
|
|---|
| 69 | #include <VBox/param.h>
|
|---|
| 70 | #include <VBox/dis.h>
|
|---|
| 71 | #include <VBox/disopcode-x86-amd64.h>
|
|---|
| 72 | #include <iprt/asm-math.h>
|
|---|
| 73 | #include <iprt/assert.h>
|
|---|
| 74 | #include <iprt/mem.h>
|
|---|
| 75 | #include <iprt/string.h>
|
|---|
| 76 | #include <iprt/x86.h>
|
|---|
| 77 |
|
|---|
| 78 | #ifndef TST_IEM_CHECK_MC
|
|---|
| 79 | # include "IEMInline.h"
|
|---|
| 80 | # include "IEMOpHlp.h"
|
|---|
| 81 | # include "IEMMc.h"
|
|---|
| 82 | #endif
|
|---|
| 83 |
|
|---|
| 84 | #include "IEMThreadedFunctions.h"
|
|---|
| 85 | #include "IEMN8veRecompiler.h" /* For a_fGstShwFlush and iemThreadedRecompilerMcDeferToCImpl0. */
|
|---|
| 86 |
|
|---|
| 87 |
|
|---|
| 88 | /*
|
|---|
| 89 | * Narrow down configs here to avoid wasting time on unused configs here.
|
|---|
| 90 | */
|
|---|
| 91 |
|
|---|
| 92 | #ifndef IEM_WITH_CODE_TLB
|
|---|
| 93 | # error The code TLB must be enabled for the recompiler.
|
|---|
| 94 | #endif
|
|---|
| 95 |
|
|---|
| 96 | #ifndef IEM_WITH_DATA_TLB
|
|---|
| 97 | # error The data TLB must be enabled for the recompiler.
|
|---|
| 98 | #endif
|
|---|
| 99 |
|
|---|
| 100 | #ifndef IEM_WITH_SETJMP
|
|---|
| 101 | # error The setjmp approach must be enabled for the recompiler.
|
|---|
| 102 | #endif
|
|---|
| 103 |
|
|---|
| 104 |
|
|---|
| 105 | /*********************************************************************************************************************************
|
|---|
| 106 | * Defined Constants And Macros *
|
|---|
| 107 | *********************************************************************************************************************************/
|
|---|
| 108 | #define g_apfnOneByteMap g_apfnIemThreadedRecompilerOneByteMap
|
|---|
| 109 | #define g_apfnTwoByteMap g_apfnIemThreadedRecompilerTwoByteMap
|
|---|
| 110 | #define g_apfnThreeByte0f3a g_apfnIemThreadedRecompilerThreeByte0f3a
|
|---|
| 111 | #define g_apfnThreeByte0f38 g_apfnIemThreadedRecompilerThreeByte0f38
|
|---|
| 112 | #define g_apfnVexMap1 g_apfnIemThreadedRecompilerVecMap1
|
|---|
| 113 | #define g_apfnVexMap2 g_apfnIemThreadedRecompilerVecMap2
|
|---|
| 114 | #define g_apfnVexMap3 g_apfnIemThreadedRecompilerVecMap3
|
|---|
| 115 |
|
|---|
| 116 |
|
|---|
| 117 | /*
|
|---|
| 118 | * Override IEM_MC_BEGIN to take down the IEM_CIMPL_F_XXX flags.
|
|---|
| 119 | */
|
|---|
| 120 | #undef IEM_MC_BEGIN
|
|---|
| 121 | #define IEM_MC_BEGIN(a_fMcFlags, a_fCImplFlags) \
|
|---|
| 122 | { \
|
|---|
| 123 | pVCpu->iem.s.fTbCurInstr = (a_fCImplFlags) /*| ((a_fMcFlags) << 20*/
|
|---|
| 124 |
|
|---|
| 125 | /*
|
|---|
| 126 | * Override IEM_MC_CALC_RM_EFF_ADDR to use iemOpHlpCalcRmEffAddrJmpEx and produce uEffAddrInfo.
|
|---|
| 127 | */
|
|---|
| 128 | #undef IEM_MC_CALC_RM_EFF_ADDR
|
|---|
| 129 | #ifndef IEM_WITH_SETJMP
|
|---|
| 130 | # define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
|
|---|
| 131 | uint64_t uEffAddrInfo; \
|
|---|
| 132 | IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddrJmpEx(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff), &uEffAddrInfo))
|
|---|
| 133 | #else
|
|---|
| 134 | # define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
|
|---|
| 135 | uint64_t uEffAddrInfo; \
|
|---|
| 136 | ((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmpEx(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &uEffAddrInfo))
|
|---|
| 137 | #endif
|
|---|
| 138 |
|
|---|
| 139 | /*
|
|---|
| 140 | * Likewise override IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES so we fetch all the opcodes.
|
|---|
| 141 | */
|
|---|
| 142 | #undef IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES
|
|---|
| 143 | #define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
|
|---|
| 144 | uint64_t uEffAddrInfo; \
|
|---|
| 145 | (void)iemOpHlpCalcRmEffAddrJmpEx(pVCpu, bRm, 0, &uEffAddrInfo); \
|
|---|
| 146 | } while (0)
|
|---|
| 147 |
|
|---|
| 148 | /*
|
|---|
| 149 | * Override the IEM_MC_REL_JMP_S*_AND_FINISH macros to check for zero byte jumps.
|
|---|
| 150 | */
|
|---|
| 151 | #undef IEM_MC_REL_JMP_S8_AND_FINISH
|
|---|
| 152 | #define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) do { \
|
|---|
| 153 | Assert(pVCpu->iem.s.fTbBranched != 0); \
|
|---|
| 154 | if ((a_i8) == 0) \
|
|---|
| 155 | pVCpu->iem.s.fTbBranched |= IEMBRANCHED_F_ZERO; \
|
|---|
| 156 | return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize); \
|
|---|
| 157 | } while (0)
|
|---|
| 158 |
|
|---|
| 159 | #undef IEM_MC_REL_JMP_S16_AND_FINISH
|
|---|
| 160 | #define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) do { \
|
|---|
| 161 | Assert(pVCpu->iem.s.fTbBranched != 0); \
|
|---|
| 162 | if ((a_i16) == 0) \
|
|---|
| 163 | pVCpu->iem.s.fTbBranched |= IEMBRANCHED_F_ZERO; \
|
|---|
| 164 | return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16)); \
|
|---|
| 165 | } while (0)
|
|---|
| 166 |
|
|---|
| 167 | #undef IEM_MC_REL_JMP_S32_AND_FINISH
|
|---|
| 168 | #define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) do { \
|
|---|
| 169 | Assert(pVCpu->iem.s.fTbBranched != 0); \
|
|---|
| 170 | if ((a_i32) == 0) \
|
|---|
| 171 | pVCpu->iem.s.fTbBranched |= IEMBRANCHED_F_ZERO; \
|
|---|
| 172 | return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize); \
|
|---|
| 173 | } while (0)
|
|---|
| 174 |
|
|---|
| 175 |
|
|---|
| 176 | /*
|
|---|
| 177 | * Emit call macros.
|
|---|
| 178 | */
|
|---|
| 179 | #define IEM_MC2_BEGIN_EMIT_CALLS(a_fCheckIrqBefore) \
|
|---|
| 180 | { \
|
|---|
| 181 | PIEMTB const pTb = pVCpu->iem.s.pCurTbR3; \
|
|---|
| 182 | uint8_t const cbInstrMc2 = IEM_GET_INSTR_LEN(pVCpu); \
|
|---|
| 183 | AssertMsg(pVCpu->iem.s.offOpcode == cbInstrMc2, \
|
|---|
| 184 | ("%u vs %u (%04x:%08RX64)\n", pVCpu->iem.s.offOpcode, cbInstrMc2, \
|
|---|
| 185 | pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); \
|
|---|
| 186 | \
|
|---|
| 187 | /* If we need to check for IRQs before the instruction, we do that before \
|
|---|
| 188 | adding any opcodes as it may abort the instruction. \
|
|---|
| 189 | Note! During compilation, we may swap IRQ and #PF exceptions here \
|
|---|
| 190 | in a manner that a real CPU would not do. However it shouldn't \
|
|---|
| 191 | be something that is easy (if at all possible) to observe in the \
|
|---|
| 192 | guest, so fine. The unexpected end-of-tb below have the same \
|
|---|
| 193 | potential "issue". */ \
|
|---|
| 194 | if (!(a_fCheckIrqBefore) || iemThreadedCompileEmitIrqCheckBefore(pVCpu, pTb)) \
|
|---|
| 195 | { /* likely */ } \
|
|---|
| 196 | else \
|
|---|
| 197 | return VINF_IEM_RECOMPILE_END_TB; \
|
|---|
| 198 | \
|
|---|
| 199 | /* No page crossing, right? */ \
|
|---|
| 200 | uint16_t const offOpcodeMc2 = pTb->cbOpcodes; \
|
|---|
| 201 | uint8_t const idxRangeMc2 = pTb->cRanges - 1; \
|
|---|
| 202 | if ( !pVCpu->iem.s.fTbCrossedPage \
|
|---|
| 203 | && !pVCpu->iem.s.fTbCheckOpcodes \
|
|---|
| 204 | && !pVCpu->iem.s.fTbBranched \
|
|---|
| 205 | && !(pTb->fFlags & IEMTB_F_CS_LIM_CHECKS)) \
|
|---|
| 206 | { \
|
|---|
| 207 | /** @todo Custom copy function, given range is 1 thru 15 bytes. */ \
|
|---|
| 208 | memcpy(&pTb->pabOpcodes[offOpcodeMc2], pVCpu->iem.s.abOpcode, pVCpu->iem.s.offOpcode); \
|
|---|
| 209 | pTb->cbOpcodes = offOpcodeMc2 + pVCpu->iem.s.offOpcode; \
|
|---|
| 210 | pTb->aRanges[idxRangeMc2].cbOpcodes += cbInstrMc2; \
|
|---|
| 211 | Assert(pTb->cbOpcodes <= pVCpu->iem.s.cbOpcodesAllocated); \
|
|---|
| 212 | } \
|
|---|
| 213 | else if (iemThreadedCompileBeginEmitCallsComplications(pVCpu, pTb)) \
|
|---|
| 214 | { /* likely */ } \
|
|---|
| 215 | else \
|
|---|
| 216 | return VINF_IEM_RECOMPILE_END_TB; \
|
|---|
| 217 | \
|
|---|
| 218 | uint8_t const idxInstrMc2 = pTb->cInstructions; \
|
|---|
| 219 | \
|
|---|
| 220 | /* Emit hardware instruction breakpoint check if enabled. */ \
|
|---|
| 221 | if (!(pTb->fFlags & IEM_F_PENDING_BRK_INSTR)) \
|
|---|
| 222 | { /* likely */ } \
|
|---|
| 223 | else \
|
|---|
| 224 | IEM_MC2_EMIT_CALL_0(kIemThreadedFunc_BltIn_CheckHwInstrBps)
|
|---|
| 225 |
|
|---|
| 226 | #define IEM_MC2_EMIT_CALL_0(a_enmFunction) do { \
|
|---|
| 227 | IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
|
|---|
| 228 | \
|
|---|
| 229 | LogFlow(("Call #%u: " #a_enmFunction "\n", pTb->Thrd.cCalls)); \
|
|---|
| 230 | PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
|
|---|
| 231 | pCall->enmFunction = a_enmFunction; \
|
|---|
| 232 | pCall->idxInstr = idxInstrMc2; \
|
|---|
| 233 | pCall->cbOpcode = cbInstrMc2; \
|
|---|
| 234 | pCall->offOpcode = offOpcodeMc2; \
|
|---|
| 235 | pCall->uTbLookup = 0; \
|
|---|
| 236 | pCall->uUnused0 = 0; \
|
|---|
| 237 | pCall->auParams[0] = 0; \
|
|---|
| 238 | pCall->auParams[1] = 0; \
|
|---|
| 239 | pCall->auParams[2] = 0; \
|
|---|
| 240 | } while (0)
|
|---|
| 241 | #define IEM_MC2_EMIT_CALL_1(a_enmFunction, a_uArg0) do { \
|
|---|
| 242 | IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
|
|---|
| 243 | uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
|
|---|
| 244 | \
|
|---|
| 245 | LogFlow(("Call #%u: " #a_enmFunction " a0=%RX64\n", pTb->Thrd.cCalls, (uint64_t)a_uArg0)); \
|
|---|
| 246 | PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
|
|---|
| 247 | pCall->enmFunction = a_enmFunction; \
|
|---|
| 248 | pCall->idxInstr = idxInstrMc2; \
|
|---|
| 249 | pCall->cbOpcode = cbInstrMc2; \
|
|---|
| 250 | pCall->offOpcode = offOpcodeMc2; \
|
|---|
| 251 | pCall->uTbLookup = 0; \
|
|---|
| 252 | pCall->uUnused0 = 0; \
|
|---|
| 253 | pCall->auParams[0] = a_uArg0; \
|
|---|
| 254 | pCall->auParams[1] = 0; \
|
|---|
| 255 | pCall->auParams[2] = 0; \
|
|---|
| 256 | } while (0)
|
|---|
| 257 | #define IEM_MC2_EMIT_CALL_2(a_enmFunction, a_uArg0, a_uArg1) do { \
|
|---|
| 258 | IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
|
|---|
| 259 | uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
|
|---|
| 260 | uint64_t const uArg1Check = (a_uArg1); RT_NOREF(uArg1Check); \
|
|---|
| 261 | \
|
|---|
| 262 | LogFlow(("Call #%u: " #a_enmFunction " a0=%RX64 a1=%RX64\n", pTb->Thrd.cCalls, (uint64_t)a_uArg0, (uint64_t)a_uArg1)); \
|
|---|
| 263 | PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
|
|---|
| 264 | pCall->enmFunction = a_enmFunction; \
|
|---|
| 265 | pCall->idxInstr = idxInstrMc2; \
|
|---|
| 266 | pCall->cbOpcode = cbInstrMc2; \
|
|---|
| 267 | pCall->offOpcode = offOpcodeMc2; \
|
|---|
| 268 | pCall->uTbLookup = 0; \
|
|---|
| 269 | pCall->uUnused0 = 0; \
|
|---|
| 270 | pCall->auParams[0] = a_uArg0; \
|
|---|
| 271 | pCall->auParams[1] = a_uArg1; \
|
|---|
| 272 | pCall->auParams[2] = 0; \
|
|---|
| 273 | } while (0)
|
|---|
| 274 | #define IEM_MC2_EMIT_CALL_3(a_enmFunction, a_uArg0, a_uArg1, a_uArg2) do { \
|
|---|
| 275 | IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
|
|---|
| 276 | uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
|
|---|
| 277 | uint64_t const uArg1Check = (a_uArg1); RT_NOREF(uArg1Check); \
|
|---|
| 278 | uint64_t const uArg2Check = (a_uArg2); RT_NOREF(uArg2Check); \
|
|---|
| 279 | \
|
|---|
| 280 | LogFlow(("Call #%u: " #a_enmFunction " a0=%RX64 a1=%RX64 a2=%RX64\n", pTb->Thrd.cCalls, (uint64_t)a_uArg0, (uint64_t)a_uArg1, (uint64_t)a_uArg2)); \
|
|---|
| 281 | PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
|
|---|
| 282 | pCall->enmFunction = a_enmFunction; \
|
|---|
| 283 | pCall->idxInstr = idxInstrMc2; \
|
|---|
| 284 | pCall->offOpcode = offOpcodeMc2; \
|
|---|
| 285 | pCall->cbOpcode = cbInstrMc2; \
|
|---|
| 286 | pCall->uTbLookup = 0; \
|
|---|
| 287 | pCall->uUnused0 = 0; \
|
|---|
| 288 | pCall->auParams[0] = a_uArg0; \
|
|---|
| 289 | pCall->auParams[1] = a_uArg1; \
|
|---|
| 290 | pCall->auParams[2] = a_uArg2; \
|
|---|
| 291 | } while (0)
|
|---|
| 292 |
|
|---|
| 293 | #define IEM_MC2_EMIT_CALL_WITH_TB_LOOKUP_0(a_fLargeTbLookup, a_enmFunction) do { \
|
|---|
| 294 | IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
|
|---|
| 295 | \
|
|---|
| 296 | LogFlow(("Call #%u: " #a_enmFunction "\n", pTb->Thrd.cCalls)); \
|
|---|
| 297 | PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
|
|---|
| 298 | pCall->enmFunction = a_enmFunction; \
|
|---|
| 299 | pCall->idxInstr = idxInstrMc2; \
|
|---|
| 300 | pCall->cbOpcode = cbInstrMc2; \
|
|---|
| 301 | pCall->offOpcode = offOpcodeMc2; \
|
|---|
| 302 | pCall->uTbLookup = IEM_TB_LOOKUP_TAB_MAKE(pTb->cTbLookupEntries, a_fLargeTbLookup); \
|
|---|
| 303 | pTb->cTbLookupEntries += !(a_fLargeTbLookup) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE; \
|
|---|
| 304 | pCall->uUnused0 = 0; \
|
|---|
| 305 | pCall->auParams[0] = 0; \
|
|---|
| 306 | pCall->auParams[1] = 0; \
|
|---|
| 307 | pCall->auParams[2] = 0; \
|
|---|
| 308 | } while (0)
|
|---|
| 309 | #define IEM_MC2_EMIT_CALL_WITH_TB_LOOKUP_1(a_fLargeTbLookup, a_enmFunction, a_uArg0) do { \
|
|---|
| 310 | IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
|
|---|
| 311 | uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
|
|---|
| 312 | \
|
|---|
| 313 | LogFlow(("Call #%u: " #a_enmFunction " a0=%RX64\n", pTb->Thrd.cCalls, (uint64_t)a_uArg0)); \
|
|---|
| 314 | PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
|
|---|
| 315 | pCall->enmFunction = a_enmFunction; \
|
|---|
| 316 | pCall->idxInstr = idxInstrMc2; \
|
|---|
| 317 | pCall->cbOpcode = cbInstrMc2; \
|
|---|
| 318 | pCall->offOpcode = offOpcodeMc2; \
|
|---|
| 319 | pCall->uTbLookup = IEM_TB_LOOKUP_TAB_MAKE(pTb->cTbLookupEntries, a_fLargeTbLookup); \
|
|---|
| 320 | pTb->cTbLookupEntries += !(a_fLargeTbLookup) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE; \
|
|---|
| 321 | pCall->uUnused0 = 0; \
|
|---|
| 322 | pCall->auParams[0] = a_uArg0; \
|
|---|
| 323 | pCall->auParams[1] = 0; \
|
|---|
| 324 | pCall->auParams[2] = 0; \
|
|---|
| 325 | } while (0)
|
|---|
| 326 | #define IEM_MC2_EMIT_CALL_WITH_TB_LOOKUP_2(a_fLargeTbLookup, a_enmFunction, a_uArg0, a_uArg1) do { \
|
|---|
| 327 | IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
|
|---|
| 328 | uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
|
|---|
| 329 | uint64_t const uArg1Check = (a_uArg1); RT_NOREF(uArg1Check); \
|
|---|
| 330 | \
|
|---|
| 331 | LogFlow(("Call #%u: " #a_enmFunction " a0=%RX64 a1=%RX64\n", pTb->Thrd.cCalls, (uint64_t)a_uArg0, (uint64_t)a_uArg1)); \
|
|---|
| 332 | PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
|
|---|
| 333 | pCall->enmFunction = a_enmFunction; \
|
|---|
| 334 | pCall->idxInstr = idxInstrMc2; \
|
|---|
| 335 | pCall->cbOpcode = cbInstrMc2; \
|
|---|
| 336 | pCall->offOpcode = offOpcodeMc2; \
|
|---|
| 337 | pCall->uTbLookup = IEM_TB_LOOKUP_TAB_MAKE(pTb->cTbLookupEntries, a_fLargeTbLookup); \
|
|---|
| 338 | pTb->cTbLookupEntries += !(a_fLargeTbLookup) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE; \
|
|---|
| 339 | pCall->uUnused0 = 0; \
|
|---|
| 340 | pCall->auParams[0] = a_uArg0; \
|
|---|
| 341 | pCall->auParams[1] = a_uArg1; \
|
|---|
| 342 | pCall->auParams[2] = 0; \
|
|---|
| 343 | } while (0)
|
|---|
| 344 | #define IEM_MC2_EMIT_CALL_WITH_TB_LOOKUP_3(a_fLargeTbLookup, a_enmFunction, a_uArg0, a_uArg1, a_uArg2) do { \
|
|---|
| 345 | IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
|
|---|
| 346 | uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
|
|---|
| 347 | uint64_t const uArg1Check = (a_uArg1); RT_NOREF(uArg1Check); \
|
|---|
| 348 | uint64_t const uArg2Check = (a_uArg2); RT_NOREF(uArg2Check); \
|
|---|
| 349 | \
|
|---|
| 350 | LogFlow(("Call #%u: " #a_enmFunction " a0=%RX64 a1=%RX64 a2=%RX64\n", pTb->Thrd.cCalls, (uint64_t)a_uArg0, (uint64_t)a_uArg1, (uint64_t)a_uArg2)); \
|
|---|
| 351 | PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
|
|---|
| 352 | pCall->enmFunction = a_enmFunction; \
|
|---|
| 353 | pCall->idxInstr = idxInstrMc2; \
|
|---|
| 354 | pCall->cbOpcode = cbInstrMc2; \
|
|---|
| 355 | pCall->offOpcode = offOpcodeMc2; \
|
|---|
| 356 | pCall->uTbLookup = IEM_TB_LOOKUP_TAB_MAKE(pTb->cTbLookupEntries, a_fLargeTbLookup); \
|
|---|
| 357 | pTb->cTbLookupEntries += !(a_fLargeTbLookup) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE; \
|
|---|
| 358 | pCall->uUnused0 = 0; \
|
|---|
| 359 | pCall->auParams[0] = a_uArg0; \
|
|---|
| 360 | pCall->auParams[1] = a_uArg1; \
|
|---|
| 361 | pCall->auParams[2] = a_uArg2; \
|
|---|
| 362 | } while (0)
|
|---|
| 363 |
|
|---|
| 364 | #define IEM_MC2_END_EMIT_CALLS(a_fCImplFlags) \
|
|---|
| 365 | Assert(pTb->cInstructions <= pTb->Thrd.cCalls); \
|
|---|
| 366 | if (pTb->cInstructions < 255) \
|
|---|
| 367 | pTb->cInstructions++; \
|
|---|
| 368 | uint32_t const fCImplFlagsMc2 = (a_fCImplFlags); \
|
|---|
| 369 | RT_NOREF(fCImplFlagsMc2); \
|
|---|
| 370 | } while (0)
|
|---|
| 371 |
|
|---|
| 372 |
|
|---|
| 373 | /*
|
|---|
| 374 | * IEM_MC_DEFER_TO_CIMPL_0 is easily wrapped up.
|
|---|
| 375 | *
|
|---|
| 376 | * Doing so will also take care of IEMOP_RAISE_DIVIDE_ERROR, IEMOP_RAISE_INVALID_LOCK_PREFIX,
|
|---|
| 377 | * IEMOP_RAISE_INVALID_OPCODE and their users.
|
|---|
| 378 | */
|
|---|
| 379 | #undef IEM_MC_DEFER_TO_CIMPL_0_RET
|
|---|
| 380 | #define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
|
|---|
| 381 | return iemThreadedRecompilerMcDeferToCImpl0(pVCpu, a_fFlags, a_fGstShwFlush, a_pfnCImpl)
|
|---|
| 382 |
|
|---|
| 383 | DECLINLINE(VBOXSTRICTRC)
|
|---|
| 384 | iemThreadedRecompilerMcDeferToCImpl0(PVMCPUCC pVCpu, uint32_t fFlags, uint64_t fGstShwFlush, PFNIEMCIMPL0 pfnCImpl)
|
|---|
| 385 | {
|
|---|
| 386 | LogFlow(("CImpl0: %04x:%08RX64 LB %#x: %#x %#RX64 %p\n",
|
|---|
| 387 | pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IEM_GET_INSTR_LEN(pVCpu), fFlags, fGstShwFlush, pfnCImpl));
|
|---|
| 388 | pVCpu->iem.s.fTbCurInstr = fFlags;
|
|---|
| 389 |
|
|---|
| 390 | IEM_MC2_BEGIN_EMIT_CALLS(fFlags & IEM_CIMPL_F_CHECK_IRQ_BEFORE);
|
|---|
| 391 | IEM_MC2_EMIT_CALL_3(kIemThreadedFunc_BltIn_DeferToCImpl0, (uintptr_t)pfnCImpl, IEM_GET_INSTR_LEN(pVCpu), fGstShwFlush);
|
|---|
| 392 | if ( (fFlags & (IEM_CIMPL_F_MODE | IEM_CIMPL_F_VMEXIT))
|
|---|
| 393 | && !(fFlags & (IEM_CIMPL_F_END_TB | IEM_CIMPL_F_BRANCH_FAR)))
|
|---|
| 394 | IEM_MC2_EMIT_CALL_1(kIemThreadedFunc_BltIn_CheckMode, pVCpu->iem.s.fExec);
|
|---|
| 395 | IEM_MC2_END_EMIT_CALLS(fFlags);
|
|---|
| 396 |
|
|---|
| 397 | /*
|
|---|
| 398 | * We have to repeat work normally done by kdCImplFlags and
|
|---|
| 399 | * ThreadedFunctionVariation.emitThreadedCallStmts here.
|
|---|
| 400 | */
|
|---|
| 401 | AssertCompile(IEM_CIMPL_F_BRANCH_DIRECT == IEMBRANCHED_F_DIRECT);
|
|---|
| 402 | AssertCompile(IEM_CIMPL_F_BRANCH_INDIRECT == IEMBRANCHED_F_INDIRECT);
|
|---|
| 403 | AssertCompile(IEM_CIMPL_F_BRANCH_RELATIVE == IEMBRANCHED_F_RELATIVE);
|
|---|
| 404 | AssertCompile(IEM_CIMPL_F_BRANCH_CONDITIONAL == IEMBRANCHED_F_CONDITIONAL);
|
|---|
| 405 | AssertCompile(IEM_CIMPL_F_BRANCH_FAR == IEMBRANCHED_F_FAR);
|
|---|
| 406 |
|
|---|
| 407 | if (fFlags & (IEM_CIMPL_F_END_TB | IEM_CIMPL_F_BRANCH_FAR))
|
|---|
| 408 | pVCpu->iem.s.fEndTb = true;
|
|---|
| 409 | else if (fFlags & IEM_CIMPL_F_BRANCH_ANY)
|
|---|
| 410 | pVCpu->iem.s.fTbBranched = fFlags & (IEM_CIMPL_F_BRANCH_ANY | IEM_CIMPL_F_BRANCH_FAR | IEM_CIMPL_F_BRANCH_CONDITIONAL);
|
|---|
| 411 |
|
|---|
| 412 | if (fFlags & IEM_CIMPL_F_CHECK_IRQ_BEFORE)
|
|---|
| 413 | pVCpu->iem.s.cInstrTillIrqCheck = 0;
|
|---|
| 414 |
|
|---|
| 415 | return pfnCImpl(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
|
|---|
| 416 | }
|
|---|
| 417 |
|
|---|
| 418 |
|
|---|
| 419 | /**
|
|---|
| 420 | * Helper for indicating that we've branched.
|
|---|
| 421 | */
|
|---|
| 422 | DECL_FORCE_INLINE(void) iemThreadedSetBranched(PVMCPUCC pVCpu, uint8_t fTbBranched)
|
|---|
| 423 | {
|
|---|
| 424 | pVCpu->iem.s.fTbBranched = fTbBranched;
|
|---|
| 425 | //pVCpu->iem.s.GCPhysTbBranchSrcBuf = pVCpu->iem.s.GCPhysInstrBuf;
|
|---|
| 426 | //pVCpu->iem.s.GCVirtTbBranchSrcBuf = pVCpu->iem.s.uInstrBufPc;
|
|---|
| 427 | }
|
|---|
| 428 |
|
|---|
| 429 |
|
|---|
| 430 | #endif /* !VMM_INCLUDED_SRC_VMMAll_IEMAllThrdTables_h */
|
|---|
