[101163] | 1 | /* $Id: IEMAllN8veRecompBltIn.cpp 104407 2024-04-23 23:16:04Z vboxsync $ */
|
---|
| 2 | /** @file
|
---|
[102634] | 3 | * IEM - Native Recompiler, Emitters for Built-In Threaded Functions.
|
---|
[101163] | 4 | */
|
---|
| 5 |
|
---|
| 6 | /*
|
---|
| 7 | * Copyright (C) 2023 Oracle and/or its affiliates.
|
---|
| 8 | *
|
---|
| 9 | * This file is part of VirtualBox base platform packages, as
|
---|
| 10 | * available from https://www.virtualbox.org.
|
---|
| 11 | *
|
---|
| 12 | * This program is free software; you can redistribute it and/or
|
---|
| 13 | * modify it under the terms of the GNU General Public License
|
---|
| 14 | * as published by the Free Software Foundation, in version 3 of the
|
---|
| 15 | * License.
|
---|
| 16 | *
|
---|
| 17 | * This program is distributed in the hope that it will be useful, but
|
---|
| 18 | * WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 19 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
---|
| 20 | * General Public License for more details.
|
---|
| 21 | *
|
---|
| 22 | * You should have received a copy of the GNU General Public License
|
---|
| 23 | * along with this program; if not, see <https://www.gnu.org/licenses>.
|
---|
| 24 | *
|
---|
| 25 | * SPDX-License-Identifier: GPL-3.0-only
|
---|
| 26 | */
|
---|
| 27 |
|
---|
| 28 |
|
---|
| 29 | /*********************************************************************************************************************************
|
---|
| 30 | * Header Files *
|
---|
| 31 | *********************************************************************************************************************************/
|
---|
[101484] | 32 | #define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
|
---|
[102695] | 33 | //#define IEM_WITH_OPAQUE_DECODER_STATE - need offCurInstrStart access for iemNativeHlpMemCodeNewPageTlbMiss and friends.
|
---|
[101163] | 34 | #define VMCPU_INCL_CPUM_GST_CTX
|
---|
[101387] | 35 | #define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
|
---|
[101163] | 36 | #include <VBox/vmm/iem.h>
|
---|
| 37 | #include <VBox/vmm/cpum.h>
|
---|
[101539] | 38 | #include <VBox/vmm/dbgf.h>
|
---|
[101163] | 39 | #include "IEMInternal.h"
|
---|
| 40 | #include <VBox/vmm/vmcc.h>
|
---|
| 41 | #include <VBox/log.h>
|
---|
| 42 | #include <VBox/err.h>
|
---|
| 43 | #include <VBox/param.h>
|
---|
| 44 | #include <iprt/assert.h>
|
---|
| 45 | #include <iprt/string.h>
|
---|
[101247] | 46 | #if defined(RT_ARCH_AMD64)
|
---|
| 47 | # include <iprt/x86.h>
|
---|
| 48 | #elif defined(RT_ARCH_ARM64)
|
---|
| 49 | # include <iprt/armv8.h>
|
---|
| 50 | #endif
|
---|
[101172] | 51 |
|
---|
[101163] | 52 |
|
---|
| 53 | #include "IEMInline.h"
|
---|
| 54 | #include "IEMThreadedFunctions.h"
|
---|
[101203] | 55 | #include "IEMN8veRecompiler.h"
|
---|
[102022] | 56 | #include "IEMN8veRecompilerEmit.h"
|
---|
[102850] | 57 | #include "IEMN8veRecompilerTlbLookup.h"
|
---|
[101163] | 58 |
|
---|
| 59 |
|
---|
| 60 |
|
---|
[101523] | 61 | /*********************************************************************************************************************************
|
---|
[102634] | 62 | * TB Helper Functions *
|
---|
[101523] | 63 | *********************************************************************************************************************************/
|
---|
[103636] | 64 | #if defined(RT_ARCH_AMD64) || defined(RT_ARCH_ARM64)
|
---|
[102663] | 65 | DECLASM(void) iemNativeHlpAsmSafeWrapLogCpuState(void);
|
---|
| 66 | #endif
|
---|
[101523] | 67 |
|
---|
| 68 |
|
---|
[102695] | 69 | /**
|
---|
| 70 | * Used by TB code to deal with a TLB miss for a new page.
|
---|
| 71 | */
|
---|
[102699] | 72 | IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCodeNewPageTlbMiss,(PVMCPUCC pVCpu))
|
---|
[102695] | 73 | {
|
---|
[102841] | 74 | STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
|
---|
[102695] | 75 | pVCpu->iem.s.pbInstrBuf = NULL;
|
---|
[102699] | 76 | pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE;
|
---|
| 77 | pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
|
---|
| 78 | iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
|
---|
| 79 | if (pVCpu->iem.s.pbInstrBuf)
|
---|
| 80 | { /* likely */ }
|
---|
| 81 | else
|
---|
| 82 | {
|
---|
| 83 | IEM_DO_LONGJMP(pVCpu, VINF_IEM_REEXEC_BREAK);
|
---|
| 84 | }
|
---|
| 85 | }
|
---|
| 86 |
|
---|
| 87 |
|
---|
| 88 | /**
|
---|
| 89 | * Used by TB code to deal with a TLB miss for a new page.
|
---|
| 90 | */
|
---|
| 91 | IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
|
---|
| 92 | {
|
---|
[102850] | 93 | STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
|
---|
[102699] | 94 | pVCpu->iem.s.pbInstrBuf = NULL;
|
---|
[102695] | 95 | pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
|
---|
| 96 | pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
|
---|
| 97 | iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
|
---|
| 98 | return pVCpu->iem.s.pbInstrBuf ? pVCpu->iem.s.GCPhysInstrBuf : NIL_RTGCPHYS;
|
---|
| 99 | }
|
---|
[101163] | 100 |
|
---|
[102695] | 101 |
|
---|
[101163] | 102 | /*********************************************************************************************************************************
|
---|
[101536] | 103 | * Builtin functions *
|
---|
[101516] | 104 | *********************************************************************************************************************************/
|
---|
| 105 |
|
---|
[101305] | 106 | /**
|
---|
[102663] | 107 | * Built-in function that does nothing.
|
---|
| 108 | *
|
---|
| 109 | * Whether this is called or not can be controlled by the entry in the
|
---|
| 110 | * IEMThreadedGenerator.katBltIns table. This can be useful to determine
|
---|
| 111 | * whether why behaviour changes when enabling the LogCpuState builtins. I.e.
|
---|
| 112 | * whether it's the reduced call count in the TBs or the threaded calls flushing
|
---|
| 113 | * register state.
|
---|
| 114 | */
|
---|
| 115 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Nop)
|
---|
| 116 | {
|
---|
| 117 | RT_NOREF(pReNative, pCallEntry);
|
---|
| 118 | return off;
|
---|
| 119 | }
|
---|
| 120 |
|
---|
[103181] | 121 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Nop)
|
---|
| 122 | {
|
---|
| 123 | *pOutgoing = *pIncoming;
|
---|
| 124 | RT_NOREF(pCallEntry);
|
---|
| 125 | }
|
---|
[102663] | 126 |
|
---|
[103181] | 127 |
|
---|
[102663] | 128 | /**
|
---|
| 129 | * Emits for for LogCpuState.
|
---|
| 130 | *
|
---|
| 131 | * This shouldn't have any relevant impact on the recompiler state.
|
---|
| 132 | */
|
---|
| 133 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_LogCpuState)
|
---|
| 134 | {
|
---|
| 135 | #ifdef RT_ARCH_AMD64
|
---|
| 136 | uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
|
---|
| 137 | /* push rax */
|
---|
| 138 | pbCodeBuf[off++] = 0x50 + X86_GREG_xAX;
|
---|
| 139 | /* push imm32 */
|
---|
| 140 | pbCodeBuf[off++] = 0x68;
|
---|
| 141 | pbCodeBuf[off++] = RT_BYTE1(pCallEntry->auParams[0]);
|
---|
| 142 | pbCodeBuf[off++] = RT_BYTE2(pCallEntry->auParams[0]);
|
---|
| 143 | pbCodeBuf[off++] = RT_BYTE3(pCallEntry->auParams[0]);
|
---|
| 144 | pbCodeBuf[off++] = RT_BYTE4(pCallEntry->auParams[0]);
|
---|
| 145 | /* mov rax, iemNativeHlpAsmSafeWrapLogCpuState */
|
---|
| 146 | pbCodeBuf[off++] = X86_OP_REX_W;
|
---|
| 147 | pbCodeBuf[off++] = 0xb8 + X86_GREG_xAX;
|
---|
| 148 | *(uint64_t *)&pbCodeBuf[off] = (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState;
|
---|
| 149 | off += sizeof(uint64_t);
|
---|
| 150 | /* call rax */
|
---|
| 151 | pbCodeBuf[off++] = 0xff;
|
---|
| 152 | pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
|
---|
| 153 | /* pop rax */
|
---|
| 154 | pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
|
---|
| 155 | /* pop rax */
|
---|
| 156 | pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
|
---|
| 157 | #else
|
---|
[103636] | 158 | off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState);
|
---|
| 159 | RT_NOREF(pCallEntry);
|
---|
[102663] | 160 | #endif
|
---|
[103636] | 161 |
|
---|
| 162 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
[102663] | 163 | return off;
|
---|
| 164 | }
|
---|
| 165 |
|
---|
[103181] | 166 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_LogCpuState)
|
---|
| 167 | {
|
---|
| 168 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 169 | RT_NOREF(pCallEntry);
|
---|
| 170 | }
|
---|
[102663] | 171 |
|
---|
[103181] | 172 |
|
---|
[102663] | 173 | /**
|
---|
[101305] | 174 | * Built-in function that calls a C-implemention function taking zero arguments.
|
---|
| 175 | */
|
---|
[102634] | 176 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_DeferToCImpl0)
|
---|
[101305] | 177 | {
|
---|
[102010] | 178 | PFNIEMCIMPL0 const pfnCImpl = (PFNIEMCIMPL0)(uintptr_t)pCallEntry->auParams[0];
|
---|
| 179 | uint8_t const cbInstr = (uint8_t)pCallEntry->auParams[1];
|
---|
[102755] | 180 | uint64_t const fGstShwFlush = pCallEntry->auParams[2];
|
---|
[102010] | 181 | return iemNativeEmitCImplCall(pReNative, off, pCallEntry->idxInstr, fGstShwFlush, (uintptr_t)pfnCImpl, cbInstr, 0, 0, 0, 0);
|
---|
[101305] | 182 | }
|
---|
[101304] | 183 |
|
---|
[103181] | 184 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_DeferToCImpl0)
|
---|
| 185 | {
|
---|
| 186 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 187 | RT_NOREF(pCallEntry);
|
---|
| 188 | }
|
---|
[101305] | 189 |
|
---|
[103181] | 190 |
|
---|
[101626] | 191 | /**
|
---|
[103649] | 192 | * Flushes pending writes in preparation of raising an exception or aborting the TB.
|
---|
| 193 | */
|
---|
| 194 | #define BODY_FLUSH_PENDING_WRITES() \
|
---|
| 195 | off = iemNativeRegFlushPendingWrites(pReNative, off);
|
---|
| 196 |
|
---|
| 197 |
|
---|
| 198 | /**
|
---|
[101640] | 199 | * Built-in function that checks for pending interrupts that can be delivered or
|
---|
| 200 | * forced action flags.
|
---|
| 201 | *
|
---|
| 202 | * This triggers after the completion of an instruction, so EIP is already at
|
---|
| 203 | * the next instruction. If an IRQ or important FF is pending, this will return
|
---|
| 204 | * a non-zero status that stops TB execution.
|
---|
| 205 | */
|
---|
[102634] | 206 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
|
---|
[101640] | 207 | {
|
---|
| 208 | RT_NOREF(pCallEntry);
|
---|
| 209 |
|
---|
[103649] | 210 | BODY_FLUSH_PENDING_WRITES();
|
---|
| 211 |
|
---|
[101640] | 212 | /* It's too convenient to use iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet below
|
---|
| 213 | and I'm too lazy to create a 'Fixed' version of that one. */
|
---|
| 214 | uint32_t const idxLabelVmCheck = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckIrq,
|
---|
| 215 | UINT32_MAX, pReNative->uCheckIrqSeqNo++);
|
---|
| 216 |
|
---|
[104407] | 217 | uint32_t const idxLabelReturnBreakFF = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnBreakFF);
|
---|
[101640] | 218 |
|
---|
| 219 | /* Again, we need to load the extended EFLAGS before we actually need them
|
---|
| 220 | in case we jump. We couldn't use iemNativeRegAllocTmpForGuestReg if we
|
---|
| 221 | loaded them inside the check, as the shadow state would not be correct
|
---|
| 222 | when the code branches before the load. Ditto PC. */
|
---|
| 223 | uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
|
---|
| 224 | kIemNativeGstRegUse_ReadOnly);
|
---|
| 225 |
|
---|
[101682] | 226 | uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ReadOnly);
|
---|
[101640] | 227 |
|
---|
| 228 | uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 229 |
|
---|
| 230 | /*
|
---|
| 231 | * Start by checking the local forced actions of the EMT we're on for IRQs
|
---|
| 232 | * and other FFs that needs servicing.
|
---|
| 233 | */
|
---|
| 234 | /** @todo this isn't even close to the NMI and interrupt conditions in EM! */
|
---|
| 235 | /* Load FFs in to idxTmpReg and AND with all relevant flags. */
|
---|
| 236 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
|
---|
| 237 | off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
|
---|
| 238 | VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
|
---|
| 239 | | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
|
---|
| 240 | | VMCPU_FF_TLB_FLUSH
|
---|
| 241 | | VMCPU_FF_UNHALT ),
|
---|
| 242 | true /*fSetFlags*/);
|
---|
| 243 | /* If we end up with ZERO in idxTmpReg there is nothing to do.*/
|
---|
| 244 | uint32_t const offFixupJumpToVmCheck1 = off;
|
---|
[102699] | 245 | off = iemNativeEmitJzToFixed(pReNative, off, off /* ASSUME jz rel8 suffices */);
|
---|
[101640] | 246 |
|
---|
| 247 | /* Some relevant FFs are set, but if's only APIC or/and PIC being set,
|
---|
| 248 | these may be supressed by EFLAGS.IF or CPUMIsInInterruptShadow. */
|
---|
| 249 | off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
|
---|
| 250 | ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC), true /*fSetFlags*/);
|
---|
| 251 | /* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
|
---|
[104407] | 252 | off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreakFF);
|
---|
[101640] | 253 |
|
---|
| 254 | /* So, it's only interrupt releated FFs and we need to see if IRQs are being
|
---|
| 255 | suppressed by the CPU or not. */
|
---|
| 256 | off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
|
---|
| 257 | off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
|
---|
[104407] | 258 | idxLabelReturnBreakFF);
|
---|
[101640] | 259 |
|
---|
| 260 | /* We've got shadow flags set, so we must check that the PC they are valid
|
---|
| 261 | for matches our current PC value. */
|
---|
| 262 | /** @todo AMD64 can do this more efficiently w/o loading uRipInhibitInt into
|
---|
| 263 | * a register. */
|
---|
| 264 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
|
---|
[104407] | 265 | off = iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, idxTmpReg, idxPcReg, idxLabelReturnBreakFF);
|
---|
[101640] | 266 |
|
---|
| 267 | /*
|
---|
| 268 | * Now check the force flags of the VM.
|
---|
| 269 | */
|
---|
| 270 | iemNativeLabelDefine(pReNative, idxLabelVmCheck, off);
|
---|
| 271 | iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
|
---|
| 272 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
|
---|
[102756] | 273 | off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
|
---|
[101640] | 274 | off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /*fSetFlags*/);
|
---|
[104407] | 275 | off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreakFF);
|
---|
[101640] | 276 |
|
---|
| 277 | /** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
|
---|
| 278 |
|
---|
| 279 | /*
|
---|
| 280 | * We're good, no IRQs or FFs pending.
|
---|
| 281 | */
|
---|
| 282 | iemNativeRegFreeTmp(pReNative, idxTmpReg);
|
---|
| 283 | iemNativeRegFreeTmp(pReNative, idxEflReg);
|
---|
| 284 | iemNativeRegFreeTmp(pReNative, idxPcReg);
|
---|
| 285 |
|
---|
| 286 | return off;
|
---|
| 287 | }
|
---|
| 288 |
|
---|
[103181] | 289 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
|
---|
| 290 | {
|
---|
| 291 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
[103318] | 292 | IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
|
---|
[103181] | 293 | RT_NOREF(pCallEntry);
|
---|
| 294 | }
|
---|
[101640] | 295 |
|
---|
[103181] | 296 |
|
---|
[101640] | 297 | /**
|
---|
[101626] | 298 | * Built-in function checks if IEMCPU::fExec has the expected value.
|
---|
| 299 | */
|
---|
[102634] | 300 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckMode)
|
---|
[101626] | 301 | {
|
---|
| 302 | uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
|
---|
[101682] | 303 | uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
|
---|
[101305] | 304 |
|
---|
[101626] | 305 | off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
|
---|
| 306 | off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
|
---|
| 307 | off = iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
|
---|
| 308 | kIemNativeLabelType_ReturnBreak);
|
---|
| 309 | iemNativeRegFreeTmp(pReNative, idxTmpReg);
|
---|
[102883] | 310 |
|
---|
| 311 | /* Maintain the recompiler fExec state. */
|
---|
| 312 | pReNative->fExec = fExpectedExec & IEMTB_F_IEM_F_MASK;
|
---|
[101626] | 313 | return off;
|
---|
| 314 | }
|
---|
| 315 |
|
---|
[103181] | 316 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckMode)
|
---|
| 317 | {
|
---|
| 318 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 319 | RT_NOREF(pCallEntry);
|
---|
| 320 | }
|
---|
[101626] | 321 |
|
---|
[103181] | 322 |
|
---|
[102593] | 323 | /**
|
---|
[102695] | 324 | * Sets idxTbCurInstr in preparation of raising an exception or aborting the TB.
|
---|
[102603] | 325 | */
|
---|
| 326 | /** @todo Optimize this, so we don't set the same value more than once. Just
|
---|
| 327 | * needs some tracking. */
|
---|
| 328 | #ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
|
---|
| 329 | # define BODY_SET_CUR_INSTR() \
|
---|
| 330 | off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, pCallEntry->idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr))
|
---|
| 331 | #else
|
---|
| 332 | # define BODY_SET_CUR_INSTR() ((void)0)
|
---|
| 333 | #endif
|
---|
| 334 |
|
---|
| 335 |
|
---|
| 336 | /**
|
---|
[102593] | 337 | * Macro that emits the 16/32-bit CS.LIM check.
|
---|
| 338 | */
|
---|
| 339 | #define BODY_CHECK_CS_LIM(a_cbInstr) \
|
---|
[102603] | 340 | off = iemNativeEmitBltInCheckCsLim(pReNative, off, (a_cbInstr))
|
---|
[101626] | 341 |
|
---|
[103181] | 342 | #define LIVENESS_CHECK_CS_LIM(a_pOutgoing) \
|
---|
| 343 | IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS)
|
---|
| 344 |
|
---|
[102593] | 345 | DECL_FORCE_INLINE(uint32_t)
|
---|
[102603] | 346 | iemNativeEmitBltInCheckCsLim(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
|
---|
[102593] | 347 | {
|
---|
| 348 | Assert(cbInstr > 0);
|
---|
| 349 | Assert(cbInstr < 16);
|
---|
[102663] | 350 | #ifdef VBOX_STRICT
|
---|
| 351 | off = iemNativeEmitMarker(pReNative, off, 0x80000001);
|
---|
| 352 | #endif
|
---|
[102593] | 353 |
|
---|
[103649] | 354 | #ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
|
---|
| 355 | Assert(pReNative->Core.offPc == 0);
|
---|
| 356 | #endif
|
---|
| 357 |
|
---|
[102593] | 358 | /*
|
---|
| 359 | * We need CS.LIM and RIP here. When cbInstr is larger than 1, we also need
|
---|
| 360 | * a temporary register for calculating the last address of the instruction.
|
---|
| 361 | *
|
---|
| 362 | * The calculation and comparisons are 32-bit. We ASSUME that the incoming
|
---|
| 363 | * RIP isn't totally invalid, i.e. that any jump/call/ret/iret instruction
|
---|
| 364 | * that last updated EIP here checked it already, and that we're therefore
|
---|
| 365 | * safe in the 32-bit wrap-around scenario to only check that the last byte
|
---|
| 366 | * is within CS.LIM. In the case of instruction-by-instruction advancing
|
---|
| 367 | * up to a EIP wrap-around, we know that CS.LIM is 4G-1 because the limit
|
---|
| 368 | * must be using 4KB granularity and the previous instruction was fine.
|
---|
| 369 | */
|
---|
| 370 | uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
|
---|
| 371 | kIemNativeGstRegUse_ReadOnly);
|
---|
| 372 | uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
|
---|
| 373 | kIemNativeGstRegUse_ReadOnly);
|
---|
| 374 | #ifdef RT_ARCH_AMD64
|
---|
[102603] | 375 | uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
|
---|
[102593] | 376 | #elif defined(RT_ARCH_ARM64)
|
---|
[102603] | 377 | uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
|
---|
[102593] | 378 | #else
|
---|
| 379 | # error "Port me"
|
---|
| 380 | #endif
|
---|
| 381 |
|
---|
| 382 | if (cbInstr != 1)
|
---|
| 383 | {
|
---|
| 384 | uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 385 |
|
---|
| 386 | /*
|
---|
| 387 | * 1. idxRegTmp = idxRegPc + cbInstr;
|
---|
| 388 | * 2. if idxRegTmp > idxRegCsLim then raise #GP(0).
|
---|
| 389 | */
|
---|
| 390 | #ifdef RT_ARCH_AMD64
|
---|
| 391 | /* 1. lea tmp32, [Pc + cbInstr - 1] */
|
---|
| 392 | if (idxRegTmp >= 8 || idxRegPc >= 8)
|
---|
| 393 | pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) | (idxRegPc < 8 ? 0 : X86_OP_REX_B);
|
---|
| 394 | pbCodeBuf[off++] = 0x8d;
|
---|
| 395 | pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, idxRegTmp & 7, idxRegPc & 7);
|
---|
| 396 | if ((idxRegPc & 7) == X86_GREG_xSP)
|
---|
| 397 | pbCodeBuf[off++] = X86_SIB_MAKE(idxRegPc & 7, 4 /*no index*/, 0);
|
---|
| 398 | pbCodeBuf[off++] = cbInstr - 1;
|
---|
| 399 |
|
---|
| 400 | /* 2. cmp tmp32(r), CsLim(r/m). */
|
---|
| 401 | if (idxRegTmp >= 8 || idxRegCsLim >= 8)
|
---|
| 402 | pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) | (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
|
---|
| 403 | pbCodeBuf[off++] = 0x3b;
|
---|
| 404 | pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegCsLim & 7);
|
---|
| 405 |
|
---|
| 406 | #elif defined(RT_ARCH_ARM64)
|
---|
| 407 | /* 1. add tmp32, Pc, #cbInstr-1 */
|
---|
| 408 | pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, idxRegTmp, idxRegPc, cbInstr - 1, false /*f64Bit*/);
|
---|
| 409 | /* 2. cmp tmp32, CsLim */
|
---|
| 410 | pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR, idxRegTmp, idxRegCsLim,
|
---|
| 411 | false /*f64Bit*/, true /*fSetFlags*/);
|
---|
| 412 |
|
---|
| 413 | #endif
|
---|
| 414 | iemNativeRegFreeTmp(pReNative, idxRegTmp);
|
---|
| 415 | }
|
---|
| 416 | else
|
---|
| 417 | {
|
---|
| 418 | /*
|
---|
| 419 | * Here we can skip step 1 and compare PC and CS.LIM directly.
|
---|
| 420 | */
|
---|
| 421 | #ifdef RT_ARCH_AMD64
|
---|
| 422 | /* 2. cmp eip(r), CsLim(r/m). */
|
---|
| 423 | if (idxRegPc >= 8 || idxRegCsLim >= 8)
|
---|
| 424 | pbCodeBuf[off++] = (idxRegPc < 8 ? 0 : X86_OP_REX_R) | (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
|
---|
| 425 | pbCodeBuf[off++] = 0x3b;
|
---|
| 426 | pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegPc & 7, idxRegCsLim & 7);
|
---|
| 427 |
|
---|
| 428 | #elif defined(RT_ARCH_ARM64)
|
---|
| 429 | /* 2. cmp Pc, CsLim */
|
---|
| 430 | pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR, idxRegPc, idxRegCsLim,
|
---|
| 431 | false /*f64Bit*/, true /*fSetFlags*/);
|
---|
| 432 |
|
---|
| 433 | #endif
|
---|
| 434 | }
|
---|
| 435 |
|
---|
| 436 | /* 3. Jump if greater. */
|
---|
| 437 | off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_RaiseGp0);
|
---|
| 438 |
|
---|
| 439 | iemNativeRegFreeTmp(pReNative, idxRegCsLim);
|
---|
| 440 | iemNativeRegFreeTmp(pReNative, idxRegPc);
|
---|
| 441 | return off;
|
---|
| 442 | }
|
---|
| 443 |
|
---|
[102603] | 444 |
|
---|
| 445 | /**
|
---|
[102624] | 446 | * Macro that considers whether we need CS.LIM checking after a branch or
|
---|
| 447 | * crossing over to a new page.
|
---|
| 448 | */
|
---|
| 449 | #define BODY_CONSIDER_CS_LIM_CHECKING(a_pTb, a_cbInstr) \
|
---|
[102663] | 450 | RT_NOREF(a_cbInstr); \
|
---|
[102624] | 451 | off = iemNativeEmitBltInConsiderLimChecking(pReNative, off)
|
---|
| 452 |
|
---|
[103181] | 453 | #define LIVENESS_CONSIDER_CS_LIM_CHECKING(a_pOutgoing) \
|
---|
| 454 | IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS); \
|
---|
| 455 | IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS)
|
---|
| 456 |
|
---|
[102624] | 457 | DECL_FORCE_INLINE(uint32_t)
|
---|
| 458 | iemNativeEmitBltInConsiderLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off)
|
---|
| 459 | {
|
---|
[102663] | 460 | #ifdef VBOX_STRICT
|
---|
| 461 | off = iemNativeEmitMarker(pReNative, off, 0x80000002);
|
---|
| 462 | #endif
|
---|
| 463 |
|
---|
[103649] | 464 | #ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
|
---|
| 465 | Assert(pReNative->Core.offPc == 0);
|
---|
| 466 | #endif
|
---|
| 467 |
|
---|
[102624] | 468 | /*
|
---|
| 469 | * This check must match the ones in the iem in iemGetTbFlagsForCurrentPc
|
---|
| 470 | * exactly:
|
---|
| 471 | *
|
---|
| 472 | * int64_t const offFromLim = (int64_t)pVCpu->cpum.GstCtx.cs.u32Limit - (int64_t)pVCpu->cpum.GstCtx.eip;
|
---|
| 473 | * if (offFromLim >= X86_PAGE_SIZE + 16 - (int32_t)(pVCpu->cpum.GstCtx.cs.u64Base & GUEST_PAGE_OFFSET_MASK))
|
---|
| 474 | * return fRet;
|
---|
| 475 | * return fRet | IEMTB_F_CS_LIM_CHECKS;
|
---|
| 476 | *
|
---|
| 477 | *
|
---|
| 478 | * We need EIP, CS.LIM and CS.BASE here.
|
---|
| 479 | */
|
---|
| 480 |
|
---|
| 481 | /* Calculate the offFromLim first: */
|
---|
| 482 | uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
|
---|
| 483 | kIemNativeGstRegUse_ReadOnly);
|
---|
| 484 | uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
|
---|
| 485 | kIemNativeGstRegUse_ReadOnly);
|
---|
| 486 | uint8_t const idxRegLeft = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 487 |
|
---|
| 488 | #ifdef RT_ARCH_ARM64
|
---|
| 489 | uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
|
---|
| 490 | pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegLeft, idxRegCsLim, idxRegPc);
|
---|
| 491 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 492 | #else
|
---|
| 493 | off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegLeft, idxRegCsLim);
|
---|
| 494 | off = iemNativeEmitSubTwoGprs(pReNative, off, idxRegLeft, idxRegPc);
|
---|
| 495 | #endif
|
---|
| 496 |
|
---|
| 497 | iemNativeRegFreeTmp(pReNative, idxRegCsLim);
|
---|
| 498 | iemNativeRegFreeTmp(pReNative, idxRegPc);
|
---|
| 499 |
|
---|
| 500 | /* Calculate the threshold level (right side). */
|
---|
| 501 | uint8_t const idxRegCsBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
|
---|
| 502 | kIemNativeGstRegUse_ReadOnly);
|
---|
| 503 | uint8_t const idxRegRight = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 504 |
|
---|
| 505 | #ifdef RT_ARCH_ARM64
|
---|
| 506 | pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
|
---|
| 507 | Assert(Armv8A64ConvertImmRImmS2Mask32(11, 0) == GUEST_PAGE_OFFSET_MASK);
|
---|
| 508 | pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegRight, idxRegCsBase, 11, 0, false /*f64Bit*/);
|
---|
| 509 | pu32CodeBuf[off++] = Armv8A64MkInstrNeg(idxRegRight);
|
---|
| 510 | pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
|
---|
| 511 | pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
|
---|
| 512 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 513 |
|
---|
| 514 | #else
|
---|
| 515 | off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegRight, GUEST_PAGE_OFFSET_MASK);
|
---|
| 516 | off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegRight, idxRegCsBase);
|
---|
| 517 | off = iemNativeEmitNegGpr(pReNative, off, idxRegRight);
|
---|
| 518 | off = iemNativeEmitAddGprImm(pReNative, off, idxRegRight, X86_PAGE_SIZE + 16);
|
---|
| 519 | #endif
|
---|
| 520 |
|
---|
| 521 | iemNativeRegFreeTmp(pReNative, idxRegCsBase);
|
---|
| 522 |
|
---|
| 523 | /* Compare the two and jump out if we're too close to the limit. */
|
---|
| 524 | off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
|
---|
| 525 | off = iemNativeEmitJlToNewLabel(pReNative, off, kIemNativeLabelType_NeedCsLimChecking);
|
---|
| 526 |
|
---|
| 527 | iemNativeRegFreeTmp(pReNative, idxRegRight);
|
---|
| 528 | iemNativeRegFreeTmp(pReNative, idxRegLeft);
|
---|
| 529 | return off;
|
---|
| 530 | }
|
---|
| 531 |
|
---|
| 532 |
|
---|
| 533 |
|
---|
| 534 | /**
|
---|
[102603] | 535 | * Macro that implements opcode (re-)checking.
|
---|
| 536 | */
|
---|
[102623] | 537 | #define BODY_CHECK_OPCODES(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
|
---|
[102663] | 538 | RT_NOREF(a_cbInstr); \
|
---|
[102603] | 539 | off = iemNativeEmitBltInCheckOpcodes(pReNative, off, (a_pTb), (a_idxRange), (a_offRange))
|
---|
| 540 |
|
---|
[103181] | 541 | #define LIVENESS_CHECK_OPCODES(a_pOutgoing) ((void)0)
|
---|
| 542 |
|
---|
[102683] | 543 | #if 0 /* debugging aid */
|
---|
| 544 | bool g_fBpOnObsoletion = false;
|
---|
| 545 | # define BP_ON_OBSOLETION g_fBpOnObsoletion
|
---|
| 546 | #else
|
---|
| 547 | # define BP_ON_OBSOLETION 0
|
---|
| 548 | #endif
|
---|
| 549 |
|
---|
[102603] | 550 | DECL_FORCE_INLINE(uint32_t)
|
---|
| 551 | iemNativeEmitBltInCheckOpcodes(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint16_t offRange)
|
---|
| 552 | {
|
---|
| 553 | Assert(idxRange < pTb->cRanges && pTb->cRanges <= RT_ELEMENTS(pTb->aRanges));
|
---|
| 554 | Assert(offRange < pTb->aRanges[idxRange].cbOpcodes);
|
---|
[102663] | 555 | #ifdef VBOX_STRICT
|
---|
| 556 | off = iemNativeEmitMarker(pReNative, off, 0x80000003);
|
---|
| 557 | #endif
|
---|
[102603] | 558 |
|
---|
| 559 | uint32_t const idxLabelObsoleteTb = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ObsoleteTb);
|
---|
| 560 |
|
---|
| 561 | /*
|
---|
| 562 | * Where to start and how much to compare.
|
---|
| 563 | *
|
---|
| 564 | * Looking at the ranges produced when r160746 was running a DOS VM with TB
|
---|
| 565 | * logging, the ranges can be anything from 1 byte to at least 0x197 bytes,
|
---|
| 566 | * with the 6, 5, 4, 7, 8, 40, 3, 2, 9 and 10 being the top 10 in the sample.
|
---|
| 567 | *
|
---|
| 568 | * The top 10 for the early boot phase of a 64-bit debian 9.4 VM: 5, 9, 8,
|
---|
| 569 | * 12, 10, 11, 6, 13, 15 and 16. Max 0x359 bytes. Same revision as above.
|
---|
| 570 | */
|
---|
| 571 | uint16_t offPage = pTb->aRanges[idxRange].offPhysPage + offRange;
|
---|
| 572 | uint16_t cbLeft = pTb->aRanges[idxRange].cbOpcodes - offRange;
|
---|
[102623] | 573 | Assert(cbLeft > 0);
|
---|
[102683] | 574 | uint8_t const *pbOpcodes = &pTb->pabOpcodes[pTb->aRanges[idxRange].offOpcodes + offRange];
|
---|
[102603] | 575 | uint32_t offConsolidatedJump = UINT32_MAX;
|
---|
| 576 |
|
---|
| 577 | #ifdef RT_ARCH_AMD64
|
---|
| 578 | /* AMD64/x86 offers a bunch of options. Smaller stuff will can be
|
---|
| 579 | completely inlined, for larger we use REPE CMPS. */
|
---|
| 580 | # define CHECK_OPCODES_CMP_IMMXX(a_idxReg, a_bOpcode) /* cost: 3 bytes */ do { \
|
---|
| 581 | pbCodeBuf[off++] = a_bOpcode; \
|
---|
| 582 | Assert(offPage < 127); \
|
---|
| 583 | pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 7, a_idxReg); \
|
---|
| 584 | pbCodeBuf[off++] = RT_BYTE1(offPage); \
|
---|
| 585 | } while (0)
|
---|
| 586 |
|
---|
| 587 | # define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
|
---|
| 588 | if (offConsolidatedJump != UINT32_MAX) \
|
---|
| 589 | { \
|
---|
| 590 | int32_t const offDisp = (int32_t)offConsolidatedJump - (int32_t)(off + 2); \
|
---|
| 591 | Assert(offDisp >= -128); \
|
---|
| 592 | pbCodeBuf[off++] = 0x75; /* jnz near */ \
|
---|
| 593 | pbCodeBuf[off++] = (uint8_t)offDisp; \
|
---|
| 594 | } \
|
---|
| 595 | else \
|
---|
| 596 | { \
|
---|
| 597 | pbCodeBuf[off++] = 0x74; /* jz near +5 */ \
|
---|
[102683] | 598 | pbCodeBuf[off++] = 0x05 + BP_ON_OBSOLETION; \
|
---|
[102603] | 599 | offConsolidatedJump = off; \
|
---|
[102683] | 600 | if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
|
---|
[102603] | 601 | pbCodeBuf[off++] = 0xe9; /* jmp rel32 */ \
|
---|
| 602 | iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_Rel32, -4); \
|
---|
| 603 | pbCodeBuf[off++] = 0x00; \
|
---|
| 604 | pbCodeBuf[off++] = 0x00; \
|
---|
| 605 | pbCodeBuf[off++] = 0x00; \
|
---|
| 606 | pbCodeBuf[off++] = 0x00; \
|
---|
| 607 | } \
|
---|
| 608 | } while (0)
|
---|
| 609 |
|
---|
| 610 | # define CHECK_OPCODES_CMP_IMM32(a_idxReg) /* cost: 3+4+2 = 9 */ do { \
|
---|
| 611 | CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
|
---|
| 612 | pbCodeBuf[off++] = *pbOpcodes++; \
|
---|
| 613 | pbCodeBuf[off++] = *pbOpcodes++; \
|
---|
| 614 | pbCodeBuf[off++] = *pbOpcodes++; \
|
---|
| 615 | pbCodeBuf[off++] = *pbOpcodes++; \
|
---|
| 616 | cbLeft -= 4; \
|
---|
| 617 | offPage += 4; \
|
---|
| 618 | CHECK_OPCODES_CMP_JMP(); \
|
---|
| 619 | } while (0)
|
---|
| 620 |
|
---|
| 621 | # define CHECK_OPCODES_CMP_IMM16(a_idxReg) /* cost: 1+3+2+2 = 8 */ do { \
|
---|
| 622 | pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP; \
|
---|
| 623 | CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
|
---|
| 624 | pbCodeBuf[off++] = *pbOpcodes++; \
|
---|
| 625 | pbCodeBuf[off++] = *pbOpcodes++; \
|
---|
| 626 | cbLeft -= 2; \
|
---|
| 627 | offPage += 2; \
|
---|
| 628 | CHECK_OPCODES_CMP_JMP(); \
|
---|
| 629 | } while (0)
|
---|
| 630 |
|
---|
| 631 | # define CHECK_OPCODES_CMP_IMM8(a_idxReg) /* cost: 3+1+2 = 6 */ do { \
|
---|
| 632 | CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x80); \
|
---|
| 633 | pbCodeBuf[off++] = *pbOpcodes++; \
|
---|
| 634 | cbLeft -= 1; \
|
---|
| 635 | offPage += 1; \
|
---|
| 636 | CHECK_OPCODES_CMP_JMP(); \
|
---|
| 637 | } while (0)
|
---|
| 638 |
|
---|
| 639 | # define CHECK_OPCODES_CMPSX(a_bOpcode, a_cbToSubtract, a_bPrefix) /* cost: 2+2 = 4 */ do { \
|
---|
| 640 | if (a_bPrefix) \
|
---|
| 641 | pbCodeBuf[off++] = (a_bPrefix); \
|
---|
| 642 | pbCodeBuf[off++] = (a_bOpcode); \
|
---|
| 643 | CHECK_OPCODES_CMP_JMP(); \
|
---|
| 644 | cbLeft -= (a_cbToSubtract); \
|
---|
| 645 | } while (0)
|
---|
| 646 |
|
---|
| 647 | # define CHECK_OPCODES_ECX_IMM(a_uValue) /* cost: 5 */ do { \
|
---|
| 648 | pbCodeBuf[off++] = 0xb8 + X86_GREG_xCX; \
|
---|
| 649 | pbCodeBuf[off++] = RT_BYTE1(a_uValue); \
|
---|
| 650 | pbCodeBuf[off++] = RT_BYTE2(a_uValue); \
|
---|
| 651 | pbCodeBuf[off++] = RT_BYTE3(a_uValue); \
|
---|
| 652 | pbCodeBuf[off++] = RT_BYTE4(a_uValue); \
|
---|
| 653 | } while (0)
|
---|
| 654 |
|
---|
| 655 | if (cbLeft <= 24)
|
---|
| 656 | {
|
---|
| 657 | uint8_t const idxRegTmp = iemNativeRegAllocTmpEx(pReNative, &off,
|
---|
| 658 | ( RT_BIT_32(X86_GREG_xAX)
|
---|
| 659 | | RT_BIT_32(X86_GREG_xCX)
|
---|
| 660 | | RT_BIT_32(X86_GREG_xDX)
|
---|
| 661 | | RT_BIT_32(X86_GREG_xBX)
|
---|
| 662 | | RT_BIT_32(X86_GREG_xSI)
|
---|
| 663 | | RT_BIT_32(X86_GREG_xDI))
|
---|
| 664 | & ~IEMNATIVE_REG_FIXED_MASK); /* pick reg not requiring rex prefix */
|
---|
| 665 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf));
|
---|
| 666 | if (offPage >= 128 - cbLeft)
|
---|
| 667 | {
|
---|
| 668 | off = iemNativeEmitAddGprImm(pReNative, off, idxRegTmp, offPage & ~(uint16_t)3);
|
---|
| 669 | offPage &= 3;
|
---|
| 670 | }
|
---|
| 671 |
|
---|
[102683] | 672 | uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 87 */);
|
---|
[102603] | 673 |
|
---|
| 674 | if (cbLeft > 8)
|
---|
| 675 | switch (offPage & 3)
|
---|
| 676 | {
|
---|
| 677 | case 0:
|
---|
| 678 | break;
|
---|
| 679 | case 1: /* cost: 6 + 8 = 14 */
|
---|
| 680 | CHECK_OPCODES_CMP_IMM8(idxRegTmp);
|
---|
| 681 | RT_FALL_THRU();
|
---|
| 682 | case 2: /* cost: 8 */
|
---|
| 683 | CHECK_OPCODES_CMP_IMM16(idxRegTmp);
|
---|
| 684 | break;
|
---|
| 685 | case 3: /* cost: 6 */
|
---|
| 686 | CHECK_OPCODES_CMP_IMM8(idxRegTmp);
|
---|
| 687 | break;
|
---|
| 688 | }
|
---|
| 689 |
|
---|
| 690 | while (cbLeft >= 4)
|
---|
| 691 | CHECK_OPCODES_CMP_IMM32(idxRegTmp); /* max iteration: 24/4 = 6; --> cost: 6 * 9 = 54 */
|
---|
| 692 |
|
---|
| 693 | if (cbLeft >= 2)
|
---|
| 694 | CHECK_OPCODES_CMP_IMM16(idxRegTmp); /* cost: 8 */
|
---|
| 695 | if (cbLeft)
|
---|
| 696 | CHECK_OPCODES_CMP_IMM8(idxRegTmp); /* cost: 6 */
|
---|
| 697 |
|
---|
| 698 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 699 | iemNativeRegFreeTmp(pReNative, idxRegTmp);
|
---|
| 700 | }
|
---|
| 701 | else
|
---|
| 702 | {
|
---|
| 703 | /* RDI = &pbInstrBuf[offPage] */
|
---|
| 704 | uint8_t const idxRegDi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xDI));
|
---|
| 705 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegDi, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
|
---|
| 706 | if (offPage != 0)
|
---|
| 707 | off = iemNativeEmitAddGprImm(pReNative, off, idxRegDi, offPage);
|
---|
| 708 |
|
---|
| 709 | /* RSI = pbOpcodes */
|
---|
| 710 | uint8_t const idxRegSi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xSI));
|
---|
| 711 | off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegSi, (uintptr_t)pbOpcodes);
|
---|
| 712 |
|
---|
| 713 | /* RCX = counts. */
|
---|
| 714 | uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
|
---|
| 715 |
|
---|
[102683] | 716 | uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /*= 35*/);
|
---|
[102603] | 717 |
|
---|
| 718 | /** @todo profile and optimize this further. Maybe an idea to align by
|
---|
| 719 | * offPage if the two cannot be reconsidled. */
|
---|
| 720 | /* Align by the page offset, so that at least one of the accesses are naturally aligned. */
|
---|
| 721 | switch (offPage & 7) /* max cost: 10 */
|
---|
| 722 | {
|
---|
| 723 | case 0:
|
---|
| 724 | break;
|
---|
| 725 | case 1: /* cost: 3+4+3 = 10 */
|
---|
| 726 | CHECK_OPCODES_CMPSX(0xa6, 1, 0);
|
---|
| 727 | RT_FALL_THRU();
|
---|
| 728 | case 2: /* cost: 4+3 = 7 */
|
---|
| 729 | CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
|
---|
| 730 | CHECK_OPCODES_CMPSX(0xa7, 4, 0);
|
---|
| 731 | break;
|
---|
| 732 | case 3: /* cost: 3+3 = 6 */
|
---|
| 733 | CHECK_OPCODES_CMPSX(0xa6, 1, 0);
|
---|
| 734 | RT_FALL_THRU();
|
---|
| 735 | case 4: /* cost: 3 */
|
---|
| 736 | CHECK_OPCODES_CMPSX(0xa7, 4, 0);
|
---|
| 737 | break;
|
---|
| 738 | case 5: /* cost: 3+4 = 7 */
|
---|
| 739 | CHECK_OPCODES_CMPSX(0xa6, 1, 0);
|
---|
| 740 | RT_FALL_THRU();
|
---|
| 741 | case 6: /* cost: 4 */
|
---|
| 742 | CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
|
---|
| 743 | break;
|
---|
| 744 | case 7: /* cost: 3 */
|
---|
| 745 | CHECK_OPCODES_CMPSX(0xa6, 1, 0);
|
---|
| 746 | break;
|
---|
| 747 | }
|
---|
| 748 |
|
---|
| 749 | /* Compare qwords: */
|
---|
| 750 | uint32_t const cQWords = cbLeft >> 3;
|
---|
| 751 | CHECK_OPCODES_ECX_IMM(cQWords); /* cost: 5 */
|
---|
| 752 |
|
---|
| 753 | pbCodeBuf[off++] = X86_OP_PRF_REPZ; /* cost: 5 */
|
---|
| 754 | CHECK_OPCODES_CMPSX(0xa7, 0, X86_OP_REX_W);
|
---|
| 755 | cbLeft &= 7;
|
---|
| 756 |
|
---|
| 757 | if (cbLeft & 4)
|
---|
[102663] | 758 | CHECK_OPCODES_CMPSX(0xa7, 4, 0); /* cost: 3 */
|
---|
[102603] | 759 | if (cbLeft & 2)
|
---|
[102663] | 760 | CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP); /* cost: 4 */
|
---|
| 761 | if (cbLeft & 1)
|
---|
| 762 | CHECK_OPCODES_CMPSX(0xa6, 1, 0); /* cost: 3 */
|
---|
[102603] | 763 |
|
---|
| 764 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 765 | iemNativeRegFreeTmp(pReNative, idxRegCx);
|
---|
| 766 | iemNativeRegFreeTmp(pReNative, idxRegSi);
|
---|
| 767 | iemNativeRegFreeTmp(pReNative, idxRegDi);
|
---|
| 768 | }
|
---|
| 769 |
|
---|
| 770 | #elif defined(RT_ARCH_ARM64)
|
---|
[102623] | 771 | /* We need pbInstrBuf in a register, whatever we do. */
|
---|
| 772 | uint8_t const idxRegSrc1Ptr = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 773 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegSrc1Ptr, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
|
---|
[102603] | 774 |
|
---|
[102623] | 775 | /* We also need at least one more register for holding bytes & words we
|
---|
| 776 | load via pbInstrBuf. */
|
---|
| 777 | uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 778 |
|
---|
| 779 | uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
|
---|
| 780 |
|
---|
| 781 | /* One byte compare can be done with the opcode byte as an immediate. We'll
|
---|
| 782 | do this to uint16_t align src1. */
|
---|
| 783 | bool fPendingJmp = RT_BOOL(offPage & 1);
|
---|
| 784 | if (fPendingJmp)
|
---|
| 785 | {
|
---|
| 786 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
|
---|
| 787 | pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, *pbOpcodes++, false /*f64Bit*/);
|
---|
| 788 | offPage += 1;
|
---|
| 789 | cbLeft -= 1;
|
---|
| 790 | }
|
---|
| 791 |
|
---|
| 792 | if (cbLeft > 0)
|
---|
| 793 | {
|
---|
| 794 | /* We need a register for holding the opcode bytes we're comparing with,
|
---|
| 795 | as CCMP only has a 5-bit immediate form and thus cannot hold bytes. */
|
---|
| 796 | uint8_t const idxRegSrc2Val = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 797 |
|
---|
| 798 | /* Word (uint32_t) aligning the src1 pointer is best done using a 16-bit constant load. */
|
---|
| 799 | if ((offPage & 3) && cbLeft >= 2)
|
---|
| 800 | {
|
---|
| 801 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 2);
|
---|
| 802 | pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
|
---|
| 803 | if (fPendingJmp)
|
---|
| 804 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 805 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
|
---|
| 806 | else
|
---|
| 807 | {
|
---|
| 808 | pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /*f64Bit*/);
|
---|
| 809 | fPendingJmp = true;
|
---|
| 810 | }
|
---|
| 811 | pbOpcodes += 2;
|
---|
| 812 | offPage += 2;
|
---|
| 813 | cbLeft -= 2;
|
---|
| 814 | }
|
---|
| 815 |
|
---|
| 816 | /* DWord (uint64_t) aligning the src2 pointer. We use a 32-bit constant here for simplicitly. */
|
---|
| 817 | if ((offPage & 7) && cbLeft >= 4)
|
---|
| 818 | {
|
---|
| 819 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 4);
|
---|
| 820 | off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
|
---|
| 821 | RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
|
---|
| 822 | if (fPendingJmp)
|
---|
| 823 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 824 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
|
---|
| 825 | else
|
---|
| 826 | {
|
---|
| 827 | pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /*f64Bit*/);
|
---|
| 828 | fPendingJmp = true;
|
---|
| 829 | }
|
---|
| 830 | pbOpcodes += 4;
|
---|
| 831 | offPage += 4;
|
---|
| 832 | cbLeft -= 4;
|
---|
| 833 | }
|
---|
| 834 |
|
---|
| 835 | /*
|
---|
| 836 | * If we've got 16 bytes or more left, switch to memcmp-style.
|
---|
| 837 | */
|
---|
| 838 | if (cbLeft >= 16)
|
---|
| 839 | {
|
---|
| 840 | /* We need a pointer to the copy of the original opcode bytes. */
|
---|
| 841 | uint8_t const idxRegSrc2Ptr = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 842 | off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Ptr, (uintptr_t)pbOpcodes);
|
---|
| 843 |
|
---|
| 844 | /* If there are more than 32 bytes to compare we create a loop, for
|
---|
| 845 | which we'll need a loop register. */
|
---|
| 846 | if (cbLeft >= 64)
|
---|
| 847 | {
|
---|
| 848 | if (fPendingJmp)
|
---|
| 849 | {
|
---|
| 850 | iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
|
---|
| 851 | pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
|
---|
| 852 | fPendingJmp = false;
|
---|
| 853 | }
|
---|
| 854 |
|
---|
| 855 | uint8_t const idxRegLoop = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 856 | uint16_t const cLoops = cbLeft / 32;
|
---|
| 857 | cbLeft = cbLeft % 32;
|
---|
| 858 | pbOpcodes += cLoops * 32;
|
---|
| 859 | pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegLoop, cLoops);
|
---|
| 860 |
|
---|
| 861 | if (offPage != 0) /** @todo optimize out this instruction. */
|
---|
| 862 | {
|
---|
| 863 | pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, offPage);
|
---|
| 864 | offPage = 0;
|
---|
| 865 | }
|
---|
| 866 |
|
---|
| 867 | uint32_t const offLoopStart = off;
|
---|
| 868 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 0);
|
---|
| 869 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 0);
|
---|
| 870 | pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
|
---|
| 871 |
|
---|
| 872 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 1);
|
---|
| 873 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 1);
|
---|
| 874 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 875 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
|
---|
| 876 |
|
---|
| 877 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 2);
|
---|
| 878 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 2);
|
---|
| 879 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 880 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
|
---|
| 881 |
|
---|
| 882 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 3);
|
---|
| 883 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 3);
|
---|
| 884 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 885 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
|
---|
| 886 |
|
---|
| 887 | iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
|
---|
| 888 | pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
|
---|
| 889 |
|
---|
| 890 | /* Advance and loop. */
|
---|
| 891 | pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, 0x20);
|
---|
| 892 | pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc2Ptr, idxRegSrc2Ptr, 0x20);
|
---|
| 893 | pu32CodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegLoop, idxRegLoop, 1, false /*f64Bit*/, true /*fSetFlags*/);
|
---|
[104030] | 894 | pu32CodeBuf[off] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, (int32_t)offLoopStart - (int32_t)off);
|
---|
| 895 | off++;
|
---|
[102623] | 896 |
|
---|
| 897 | iemNativeRegFreeTmp(pReNative, idxRegLoop);
|
---|
| 898 | }
|
---|
| 899 |
|
---|
| 900 | /* Deal with any remaining dwords (uint64_t). There can be up to
|
---|
| 901 | three if we looped and four if we didn't. */
|
---|
| 902 | uint32_t offSrc2 = 0;
|
---|
| 903 | while (cbLeft >= 8)
|
---|
| 904 | {
|
---|
| 905 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val,
|
---|
| 906 | idxRegSrc1Ptr, offPage / 8);
|
---|
| 907 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val,
|
---|
| 908 | idxRegSrc2Ptr, offSrc2 / 8);
|
---|
| 909 | if (fPendingJmp)
|
---|
| 910 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 911 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
|
---|
| 912 | else
|
---|
| 913 | {
|
---|
| 914 | pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
|
---|
| 915 | fPendingJmp = true;
|
---|
| 916 | }
|
---|
| 917 | pbOpcodes += 8;
|
---|
| 918 | offPage += 8;
|
---|
| 919 | offSrc2 += 8;
|
---|
| 920 | cbLeft -= 8;
|
---|
| 921 | }
|
---|
| 922 |
|
---|
| 923 | iemNativeRegFreeTmp(pReNative, idxRegSrc2Ptr);
|
---|
| 924 | /* max cost thus far: memcmp-loop=43 vs memcmp-no-loop=30 */
|
---|
| 925 | }
|
---|
| 926 | /*
|
---|
| 927 | * Otherwise, we compare with constants and merge with the general mop-up.
|
---|
| 928 | */
|
---|
| 929 | else
|
---|
| 930 | {
|
---|
| 931 | while (cbLeft >= 8)
|
---|
| 932 | {
|
---|
| 933 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr,
|
---|
| 934 | offPage / 8);
|
---|
| 935 | off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Val,
|
---|
| 936 | RT_MAKE_U64_FROM_MSB_U8(pbOpcodes[7], pbOpcodes[6], pbOpcodes[5], pbOpcodes[4],
|
---|
| 937 | pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
|
---|
| 938 | if (fPendingJmp)
|
---|
| 939 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 940 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, true /*f64Bit*/);
|
---|
| 941 | else
|
---|
| 942 | {
|
---|
| 943 | pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, true /*f64Bit*/);
|
---|
| 944 | fPendingJmp = true;
|
---|
| 945 | }
|
---|
| 946 | pbOpcodes += 8;
|
---|
| 947 | offPage += 8;
|
---|
| 948 | cbLeft -= 8;
|
---|
| 949 | }
|
---|
| 950 | /* max cost thus far: 21 */
|
---|
| 951 | }
|
---|
| 952 |
|
---|
| 953 | /* Deal with any remaining bytes (7 or less). */
|
---|
| 954 | Assert(cbLeft < 8);
|
---|
| 955 | if (cbLeft >= 4)
|
---|
| 956 | {
|
---|
| 957 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr,
|
---|
| 958 | offPage / 4);
|
---|
| 959 | off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
|
---|
| 960 | RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
|
---|
| 961 | if (fPendingJmp)
|
---|
| 962 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 963 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
|
---|
| 964 | else
|
---|
| 965 | {
|
---|
| 966 | pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /*f64Bit*/);
|
---|
| 967 | fPendingJmp = true;
|
---|
| 968 | }
|
---|
| 969 | pbOpcodes += 4;
|
---|
| 970 | offPage += 4;
|
---|
| 971 | cbLeft -= 4;
|
---|
| 972 |
|
---|
| 973 | }
|
---|
| 974 |
|
---|
| 975 | if (cbLeft >= 2)
|
---|
| 976 | {
|
---|
| 977 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr,
|
---|
| 978 | offPage / 2);
|
---|
| 979 | pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
|
---|
| 980 | if (fPendingJmp)
|
---|
| 981 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 982 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
|
---|
| 983 | else
|
---|
| 984 | {
|
---|
| 985 | pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /*f64Bit*/);
|
---|
| 986 | fPendingJmp = true;
|
---|
| 987 | }
|
---|
| 988 | pbOpcodes += 2;
|
---|
| 989 | offPage += 2;
|
---|
| 990 | cbLeft -= 2;
|
---|
| 991 | }
|
---|
| 992 |
|
---|
| 993 | if (cbLeft > 0)
|
---|
| 994 | {
|
---|
| 995 | Assert(cbLeft == 1);
|
---|
| 996 | pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
|
---|
| 997 | if (fPendingJmp)
|
---|
| 998 | {
|
---|
| 999 | pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, pbOpcodes[0]);
|
---|
| 1000 | pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
|
---|
| 1001 | ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
|
---|
| 1002 | }
|
---|
| 1003 | else
|
---|
| 1004 | {
|
---|
| 1005 | pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes[0], false /*f64Bit*/);
|
---|
| 1006 | fPendingJmp = true;
|
---|
| 1007 | }
|
---|
| 1008 | pbOpcodes += 1;
|
---|
| 1009 | offPage += 1;
|
---|
| 1010 | cbLeft -= 1;
|
---|
| 1011 | }
|
---|
| 1012 |
|
---|
| 1013 | iemNativeRegFreeTmp(pReNative, idxRegSrc2Val);
|
---|
| 1014 | }
|
---|
| 1015 | Assert(cbLeft == 0);
|
---|
| 1016 |
|
---|
| 1017 | /*
|
---|
| 1018 | * Finally, the branch on difference.
|
---|
| 1019 | */
|
---|
| 1020 | if (fPendingJmp)
|
---|
| 1021 | {
|
---|
| 1022 | iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
|
---|
| 1023 | pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
|
---|
| 1024 | }
|
---|
| 1025 | RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump, idxLabelObsoleteTb);
|
---|
| 1026 |
|
---|
| 1027 | /* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
|
---|
| 1028 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1029 | iemNativeRegFreeTmp(pReNative, idxRegSrc1Val);
|
---|
| 1030 | iemNativeRegFreeTmp(pReNative, idxRegSrc1Ptr);
|
---|
| 1031 |
|
---|
[102603] | 1032 | #else
|
---|
| 1033 | # error "Port me"
|
---|
| 1034 | #endif
|
---|
| 1035 | return off;
|
---|
| 1036 | }
|
---|
| 1037 |
|
---|
| 1038 |
|
---|
[102663] | 1039 | /** Duplicated in IEMAllThrdFuncsBltIn.cpp. */
|
---|
| 1040 | DECL_FORCE_INLINE(RTGCPHYS) iemTbGetRangePhysPageAddr(PCIEMTB pTb, uint8_t idxRange)
|
---|
| 1041 | {
|
---|
| 1042 | Assert(idxRange < RT_MIN(pTb->cRanges, RT_ELEMENTS(pTb->aRanges)));
|
---|
| 1043 | uint8_t const idxPage = pTb->aRanges[idxRange].idxPhysPage;
|
---|
| 1044 | Assert(idxPage <= RT_ELEMENTS(pTb->aGCPhysPages));
|
---|
| 1045 | if (idxPage == 0)
|
---|
| 1046 | return pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
|
---|
| 1047 | Assert(!(pTb->aGCPhysPages[idxPage - 1] & GUEST_PAGE_OFFSET_MASK));
|
---|
| 1048 | return pTb->aGCPhysPages[idxPage - 1];
|
---|
| 1049 | }
|
---|
| 1050 |
|
---|
| 1051 |
|
---|
| 1052 | /**
|
---|
| 1053 | * Macro that implements PC check after a conditional branch.
|
---|
| 1054 | */
|
---|
| 1055 | #define BODY_CHECK_PC_AFTER_BRANCH(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
|
---|
| 1056 | RT_NOREF(a_cbInstr); \
|
---|
| 1057 | off = iemNativeEmitBltInCheckPcAfterBranch(pReNative, off, a_pTb, a_idxRange, a_offRange)
|
---|
| 1058 |
|
---|
[103181] | 1059 | #define LIVENESS_CHECK_PC_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
|
---|
| 1060 | if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
|
---|
| 1061 | IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
|
---|
| 1062 | else do { } while (0)
|
---|
| 1063 |
|
---|
[102663] | 1064 | DECL_FORCE_INLINE(uint32_t)
|
---|
| 1065 | iemNativeEmitBltInCheckPcAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb,
|
---|
| 1066 | uint8_t idxRange, uint16_t offRange)
|
---|
| 1067 | {
|
---|
| 1068 | #ifdef VBOX_STRICT
|
---|
| 1069 | off = iemNativeEmitMarker(pReNative, off, 0x80000004);
|
---|
| 1070 | #endif
|
---|
| 1071 |
|
---|
[103649] | 1072 | #ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
|
---|
| 1073 | Assert(pReNative->Core.offPc == 0);
|
---|
| 1074 | #endif
|
---|
| 1075 |
|
---|
[102663] | 1076 | /*
|
---|
| 1077 | * The GCPhysRangePageWithOffset value in the threaded function is a fixed
|
---|
| 1078 | * constant for us here.
|
---|
| 1079 | *
|
---|
| 1080 | * We can pretend that iem.s.cbInstrBufTotal is X86_PAGE_SIZE here, because
|
---|
| 1081 | * it serves no purpose as a CS.LIM, if that's needed we've just performed
|
---|
| 1082 | * it, and as long as we don't implement code TLB reload code here there is
|
---|
| 1083 | * no point in checking that the TLB data we're using is still valid.
|
---|
| 1084 | *
|
---|
| 1085 | * What we to do is.
|
---|
| 1086 | * 1. Calculate the FLAT PC (RIP + CS.BASE).
|
---|
| 1087 | * 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
|
---|
| 1088 | * 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
|
---|
| 1089 | * we're in the wrong spot and need to find a new TB.
|
---|
| 1090 | * 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
|
---|
| 1091 | * GCPhysRangePageWithOffset constant mentioned above.
|
---|
| 1092 | *
|
---|
| 1093 | * The adding of CS.BASE to RIP can be skipped in the first step if we're
|
---|
| 1094 | * in 64-bit code or flat 32-bit.
|
---|
| 1095 | */
|
---|
| 1096 |
|
---|
| 1097 | /* Allocate registers for step 1. Get the shadowed stuff before allocating
|
---|
| 1098 | the temp register, so we don't accidentally clobber something we'll be
|
---|
| 1099 | needing again immediately. This is why we get idxRegCsBase here. */
|
---|
| 1100 | uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
|
---|
| 1101 | kIemNativeGstRegUse_ReadOnly);
|
---|
| 1102 | uint8_t const idxRegCsBase = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX
|
---|
| 1103 | : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
|
---|
| 1104 | kIemNativeGstRegUse_ReadOnly);
|
---|
| 1105 |
|
---|
| 1106 | uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 1107 |
|
---|
| 1108 | #ifdef VBOX_STRICT
|
---|
| 1109 | /* Do assertions before idxRegTmp contains anything. */
|
---|
| 1110 | Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
|
---|
| 1111 | # ifdef RT_ARCH_AMD64
|
---|
| 1112 | {
|
---|
| 1113 | uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
|
---|
| 1114 | /* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
|
---|
| 1115 | if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
|
---|
| 1116 | {
|
---|
| 1117 | /* cmp r/m64, imm8 */
|
---|
| 1118 | pbCodeBuf[off++] = X86_OP_REX_W;
|
---|
| 1119 | pbCodeBuf[off++] = 0x83;
|
---|
| 1120 | off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
|
---|
| 1121 | pbCodeBuf[off++] = 0;
|
---|
| 1122 | /* je rel8 */
|
---|
| 1123 | pbCodeBuf[off++] = 0x74;
|
---|
| 1124 | pbCodeBuf[off++] = 1;
|
---|
| 1125 | /* int3 */
|
---|
| 1126 | pbCodeBuf[off++] = 0xcc;
|
---|
| 1127 |
|
---|
| 1128 | }
|
---|
| 1129 |
|
---|
| 1130 | /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
|
---|
| 1131 | /* test r/m64, imm32 */
|
---|
| 1132 | pbCodeBuf[off++] = X86_OP_REX_W;
|
---|
| 1133 | pbCodeBuf[off++] = 0xf7;
|
---|
| 1134 | off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
|
---|
| 1135 | pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
|
---|
| 1136 | pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
|
---|
| 1137 | pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
|
---|
| 1138 | pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
|
---|
| 1139 | /* jz rel8 */
|
---|
| 1140 | pbCodeBuf[off++] = 0x74;
|
---|
| 1141 | pbCodeBuf[off++] = 1;
|
---|
| 1142 | /* int3 */
|
---|
| 1143 | pbCodeBuf[off++] = 0xcc;
|
---|
| 1144 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1145 | }
|
---|
| 1146 | # else
|
---|
| 1147 |
|
---|
| 1148 | /* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
|
---|
| 1149 | if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
|
---|
| 1150 | {
|
---|
| 1151 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
|
---|
| 1152 | # ifdef RT_ARCH_ARM64
|
---|
| 1153 | uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
|
---|
[102685] | 1154 | pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 2, idxRegTmp);
|
---|
[102663] | 1155 | pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2004);
|
---|
| 1156 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1157 | # else
|
---|
| 1158 | # error "Port me!"
|
---|
| 1159 | # endif
|
---|
| 1160 | }
|
---|
| 1161 | # endif
|
---|
| 1162 |
|
---|
| 1163 | #endif /* VBOX_STRICT */
|
---|
| 1164 |
|
---|
| 1165 | /* 1+2. Calculate 'off' first (into idxRegTmp). */
|
---|
| 1166 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
|
---|
| 1167 | if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
|
---|
| 1168 | {
|
---|
| 1169 | #ifdef RT_ARCH_ARM64
|
---|
| 1170 | uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
|
---|
| 1171 | pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
|
---|
| 1172 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1173 | #else
|
---|
| 1174 | off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
|
---|
| 1175 | off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
|
---|
| 1176 | #endif
|
---|
| 1177 | }
|
---|
| 1178 | else
|
---|
| 1179 | {
|
---|
| 1180 | #ifdef RT_ARCH_ARM64
|
---|
| 1181 | uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
|
---|
| 1182 | pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
|
---|
| 1183 | pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
|
---|
| 1184 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1185 | #else
|
---|
| 1186 | off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
|
---|
| 1187 | off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
|
---|
| 1188 | off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
|
---|
| 1189 | #endif
|
---|
| 1190 | iemNativeRegFreeTmp(pReNative, idxRegCsBase);
|
---|
| 1191 | }
|
---|
| 1192 | iemNativeRegFreeTmp(pReNative, idxRegPc);
|
---|
| 1193 |
|
---|
| 1194 | /* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
|
---|
| 1195 | off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
|
---|
| 1196 | off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
|
---|
| 1197 |
|
---|
| 1198 | /* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
|
---|
| 1199 | #ifdef RT_ARCH_AMD64
|
---|
| 1200 | uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
|
---|
| 1201 | pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
|
---|
| 1202 | pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
|
---|
| 1203 | off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
|
---|
| 1204 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1205 |
|
---|
| 1206 | #elif defined(RT_ARCH_ARM64)
|
---|
| 1207 | uint8_t const idxRegTmp2 = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 1208 |
|
---|
| 1209 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
|
---|
| 1210 | uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
|
---|
[102685] | 1211 | pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
|
---|
[102663] | 1212 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1213 |
|
---|
| 1214 | # ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
|
---|
[102685] | 1215 | off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /*fSetFlags*/);
|
---|
[102701] | 1216 | off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
|
---|
[102663] | 1217 | off = iemNativeEmitBrk(pReNative, off, 0x2005);
|
---|
| 1218 | # endif
|
---|
| 1219 | iemNativeRegFreeTmp(pReNative, idxRegTmp2);
|
---|
| 1220 | #else
|
---|
| 1221 | # error "Port me"
|
---|
| 1222 | #endif
|
---|
| 1223 |
|
---|
| 1224 | RTGCPHYS const GCPhysRangePageWithOffset = ( iemTbGetRangePhysPageAddr(pTb, idxRange)
|
---|
| 1225 | | pTb->aRanges[idxRange].offPhysPage)
|
---|
| 1226 | + offRange;
|
---|
| 1227 | off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
|
---|
| 1228 | kIemNativeLabelType_CheckBranchMiss);
|
---|
| 1229 |
|
---|
| 1230 | iemNativeRegFreeTmp(pReNative, idxRegTmp);
|
---|
| 1231 | return off;
|
---|
| 1232 | }
|
---|
| 1233 |
|
---|
| 1234 |
|
---|
[102695] | 1235 | /**
|
---|
| 1236 | * Macro that implements TLB loading and updating pbInstrBuf updating for an
|
---|
| 1237 | * instruction crossing into a new page.
|
---|
| 1238 | *
|
---|
| 1239 | * This may long jump if we're raising a \#PF, \#GP or similar trouble.
|
---|
| 1240 | */
|
---|
| 1241 | #define BODY_LOAD_TLB_FOR_NEW_PAGE(a_pTb, a_offInstr, a_idxRange, a_cbInstr) \
|
---|
| 1242 | RT_NOREF(a_cbInstr); \
|
---|
| 1243 | off = iemNativeEmitBltLoadTlbForNewPage(pReNative, off, pTb, a_idxRange, a_offInstr)
|
---|
| 1244 |
|
---|
[103181] | 1245 | #define LIVENESS_LOAD_TLB_FOR_NEW_PAGE(a_pOutgoing, a_pCallEntry) \
|
---|
| 1246 | if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
|
---|
| 1247 | IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
|
---|
| 1248 | else do { } while (0)
|
---|
| 1249 |
|
---|
[102695] | 1250 | DECL_FORCE_INLINE(uint32_t)
|
---|
| 1251 | iemNativeEmitBltLoadTlbForNewPage(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint8_t offInstr)
|
---|
| 1252 | {
|
---|
| 1253 | #ifdef VBOX_STRICT
|
---|
| 1254 | off = iemNativeEmitMarker(pReNative, off, 0x80000005);
|
---|
| 1255 | #endif
|
---|
| 1256 |
|
---|
| 1257 | /*
|
---|
| 1258 | * Define labels and allocate the register for holding the GCPhys of the new page.
|
---|
| 1259 | */
|
---|
[102850] | 1260 | uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
|
---|
| 1261 | uint32_t const idxRegGCPhys = iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 1262 | IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
|
---|
| 1263 | uint32_t const idxLabelTlbLookup = !TlbState.fSkip
|
---|
| 1264 | ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
|
---|
| 1265 | : UINT32_MAX;
|
---|
[102695] | 1266 |
|
---|
[102850] | 1267 | //off = iemNativeEmitBrk(pReNative, off, 0x1111);
|
---|
| 1268 |
|
---|
[102695] | 1269 | /*
|
---|
[102850] | 1270 | * Jump to the TLB lookup code.
|
---|
[102695] | 1271 | */
|
---|
[102850] | 1272 | if (!TlbState.fSkip)
|
---|
| 1273 | off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
|
---|
[102695] | 1274 |
|
---|
| 1275 | /*
|
---|
[102850] | 1276 | * TlbMiss:
|
---|
| 1277 | *
|
---|
| 1278 | * Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
|
---|
[102695] | 1279 | */
|
---|
[102850] | 1280 | uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
|
---|
[102695] | 1281 |
|
---|
[102846] | 1282 | /* Save variables in volatile registers. */
|
---|
[102850] | 1283 | uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() | RT_BIT_32(idxRegGCPhys);
|
---|
[102846] | 1284 | off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
|
---|
| 1285 |
|
---|
[102695] | 1286 | /* IEMNATIVE_CALL_ARG1_GREG = offInstr */
|
---|
| 1287 | off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offInstr);
|
---|
| 1288 |
|
---|
| 1289 | /* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
|
---|
| 1290 | off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
|
---|
| 1291 |
|
---|
| 1292 | /* Done setting up parameters, make the call. */
|
---|
[102699] | 1293 | off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMissWithOff);
|
---|
[102695] | 1294 |
|
---|
| 1295 | /* Move the result to the right register. */
|
---|
| 1296 | if (idxRegGCPhys != IEMNATIVE_CALL_RET_GREG)
|
---|
| 1297 | off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegGCPhys, IEMNATIVE_CALL_RET_GREG);
|
---|
| 1298 |
|
---|
[102846] | 1299 | /* Restore variables and guest shadow registers to volatile registers. */
|
---|
| 1300 | off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
|
---|
[102850] | 1301 | off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /*fCode*/));
|
---|
[102846] | 1302 |
|
---|
[102850] | 1303 | #ifdef IEMNATIVE_WITH_TLB_LOOKUP
|
---|
| 1304 | if (!TlbState.fSkip)
|
---|
| 1305 | {
|
---|
| 1306 | /* end of TlbMiss - Jump to the done label. */
|
---|
| 1307 | uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
|
---|
| 1308 | off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
|
---|
[102695] | 1309 |
|
---|
[102850] | 1310 | /*
|
---|
| 1311 | * TlbLookup:
|
---|
| 1312 | */
|
---|
| 1313 | off = iemNativeEmitTlbLookup<false>(pReNative, off, &TlbState,
|
---|
| 1314 | IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
|
---|
| 1315 | 1 /*cbMem*/, 0 /*fAlignMask*/, IEM_ACCESS_TYPE_EXEC,
|
---|
| 1316 | idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
|
---|
| 1317 |
|
---|
| 1318 | # ifdef VBOX_WITH_STATISTICS
|
---|
| 1319 | off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
|
---|
| 1320 | RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
|
---|
| 1321 | # endif
|
---|
| 1322 |
|
---|
| 1323 | /*
|
---|
| 1324 | * TlbDone:
|
---|
| 1325 | */
|
---|
| 1326 | iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
|
---|
| 1327 | TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /*idxVarGCPtrMem*/, true /*fIsCode*/);
|
---|
| 1328 | }
|
---|
| 1329 | #else
|
---|
| 1330 | RT_NOREF(idxLabelTlbMiss);
|
---|
| 1331 | #endif
|
---|
| 1332 |
|
---|
[102695] | 1333 | /*
|
---|
| 1334 | * Now check the physical address of the page matches the expected one.
|
---|
| 1335 | */
|
---|
| 1336 | RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
|
---|
| 1337 | off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegGCPhys, GCPhysNewPage,
|
---|
| 1338 | kIemNativeLabelType_ObsoleteTb);
|
---|
| 1339 |
|
---|
| 1340 | iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
|
---|
| 1341 | return off;
|
---|
| 1342 | }
|
---|
| 1343 |
|
---|
| 1344 |
|
---|
[102699] | 1345 | /**
|
---|
| 1346 | * Macro that implements TLB loading and updating pbInstrBuf updating when
|
---|
| 1347 | * branching or when crossing a page on an instruction boundrary.
|
---|
| 1348 | *
|
---|
| 1349 | * This differs from BODY_LOAD_TLB_FOR_NEW_PAGE in that it will first check if
|
---|
| 1350 | * it is an inter-page branch and also check the page offset.
|
---|
| 1351 | *
|
---|
| 1352 | * This may long jump if we're raising a \#PF, \#GP or similar trouble.
|
---|
| 1353 | */
|
---|
| 1354 | #define BODY_LOAD_TLB_AFTER_BRANCH(a_pTb, a_idxRange, a_cbInstr) \
|
---|
| 1355 | RT_NOREF(a_cbInstr); \
|
---|
| 1356 | off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
|
---|
| 1357 |
|
---|
[103181] | 1358 | #define LIVENESS_LOAD_TLB_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
|
---|
| 1359 | if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
|
---|
| 1360 | IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
|
---|
| 1361 | else do { } while (0)
|
---|
| 1362 |
|
---|
[102699] | 1363 | DECL_FORCE_INLINE(uint32_t)
|
---|
| 1364 | iemNativeEmitBltLoadTlbAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange)
|
---|
| 1365 | {
|
---|
| 1366 | #ifdef VBOX_STRICT
|
---|
| 1367 | off = iemNativeEmitMarker(pReNative, off, 0x80000006);
|
---|
| 1368 | #endif
|
---|
| 1369 |
|
---|
[103649] | 1370 | BODY_FLUSH_PENDING_WRITES();
|
---|
| 1371 |
|
---|
[102699] | 1372 | /*
|
---|
| 1373 | * Define labels and allocate the register for holding the GCPhys of the new page.
|
---|
| 1374 | */
|
---|
[102856] | 1375 | uint32_t const idxLabelCheckBranchMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckBranchMiss);
|
---|
| 1376 | uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
|
---|
[102699] | 1377 | RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
|
---|
| 1378 | | pTb->aRanges[idxRange].offPhysPage;
|
---|
| 1379 |
|
---|
| 1380 | /*
|
---|
| 1381 | *
|
---|
| 1382 | * First check if RIP is within the current code.
|
---|
| 1383 | *
|
---|
| 1384 | * This is very similar to iemNativeEmitBltInCheckPcAfterBranch, the only
|
---|
| 1385 | * difference is what we do when stuff doesn't match up.
|
---|
| 1386 | *
|
---|
| 1387 | * What we to do is.
|
---|
| 1388 | * 1. Calculate the FLAT PC (RIP + CS.BASE).
|
---|
| 1389 | * 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
|
---|
| 1390 | * 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
|
---|
| 1391 | * we need to retranslate RIP via the TLB.
|
---|
| 1392 | * 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
|
---|
| 1393 | * GCPhysRangePageWithOffset constant mentioned above.
|
---|
| 1394 | *
|
---|
| 1395 | * The adding of CS.BASE to RIP can be skipped in the first step if we're
|
---|
| 1396 | * in 64-bit code or flat 32-bit.
|
---|
| 1397 | *
|
---|
| 1398 | */
|
---|
| 1399 |
|
---|
| 1400 | /* Allocate registers for step 1. Get the shadowed stuff before allocating
|
---|
| 1401 | the temp register, so we don't accidentally clobber something we'll be
|
---|
[102856] | 1402 | needing again immediately. This is why we get idxRegCsBase here.
|
---|
| 1403 | Update: We share registers with the TlbState, as the TLB code path has
|
---|
| 1404 | little in common with the rest of the code. */
|
---|
| 1405 | bool const fIsFlat = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec);
|
---|
| 1406 | IEMNATIVEEMITTLBSTATE const TlbState(pReNative, fIsFlat, &off);
|
---|
| 1407 | uint8_t const idxRegPc = !TlbState.fSkip ? TlbState.idxRegPtr
|
---|
| 1408 | : iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
|
---|
[102699] | 1409 | kIemNativeGstRegUse_ReadOnly, true /*fNoVolatileRegs*/);
|
---|
[102856] | 1410 | uint8_t const idxRegCsBase = !TlbState.fSkip || fIsFlat ? TlbState.idxRegSegBase
|
---|
[102699] | 1411 | : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
|
---|
| 1412 | kIemNativeGstRegUse_ReadOnly, true /*fNoVolatileRegs*/);
|
---|
| 1413 |
|
---|
[102856] | 1414 | uint8_t const idxRegTmp = !TlbState.fSkip ? TlbState.idxReg1 : iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 1415 | uint8_t const idxRegTmp2 = !TlbState.fSkip ? TlbState.idxReg2 : iemNativeRegAllocTmp(pReNative, &off);
|
---|
| 1416 | uint8_t const idxRegDummy = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
|
---|
[102699] | 1417 |
|
---|
| 1418 | #ifdef VBOX_STRICT
|
---|
| 1419 | /* Do assertions before idxRegTmp contains anything. */
|
---|
| 1420 | Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
|
---|
| 1421 | # ifdef RT_ARCH_AMD64
|
---|
| 1422 | {
|
---|
| 1423 | uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
|
---|
| 1424 | /* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
|
---|
| 1425 | if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
|
---|
| 1426 | {
|
---|
| 1427 | /* cmp r/m64, imm8 */
|
---|
| 1428 | pbCodeBuf[off++] = X86_OP_REX_W;
|
---|
| 1429 | pbCodeBuf[off++] = 0x83;
|
---|
| 1430 | off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
|
---|
| 1431 | pbCodeBuf[off++] = 0;
|
---|
| 1432 | /* je rel8 */
|
---|
| 1433 | pbCodeBuf[off++] = 0x74;
|
---|
| 1434 | pbCodeBuf[off++] = 1;
|
---|
| 1435 | /* int3 */
|
---|
| 1436 | pbCodeBuf[off++] = 0xcc;
|
---|
| 1437 |
|
---|
| 1438 | }
|
---|
| 1439 |
|
---|
| 1440 | /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
|
---|
| 1441 | /* test r/m64, imm32 */
|
---|
| 1442 | pbCodeBuf[off++] = X86_OP_REX_W;
|
---|
| 1443 | pbCodeBuf[off++] = 0xf7;
|
---|
| 1444 | off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
|
---|
| 1445 | pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
|
---|
| 1446 | pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
|
---|
| 1447 | pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
|
---|
| 1448 | pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
|
---|
| 1449 | /* jz rel8 */
|
---|
| 1450 | pbCodeBuf[off++] = 0x74;
|
---|
| 1451 | pbCodeBuf[off++] = 1;
|
---|
| 1452 | /* int3 */
|
---|
| 1453 | pbCodeBuf[off++] = 0xcc;
|
---|
| 1454 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1455 | }
|
---|
| 1456 | # else
|
---|
| 1457 |
|
---|
| 1458 | /* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
|
---|
| 1459 | if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
|
---|
| 1460 | {
|
---|
| 1461 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
|
---|
| 1462 | # ifdef RT_ARCH_ARM64
|
---|
| 1463 | uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
|
---|
| 1464 | pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 2, idxRegTmp);
|
---|
| 1465 | pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2006);
|
---|
| 1466 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1467 | # else
|
---|
| 1468 | # error "Port me!"
|
---|
| 1469 | # endif
|
---|
| 1470 | }
|
---|
| 1471 | # endif
|
---|
| 1472 |
|
---|
| 1473 | #endif /* VBOX_STRICT */
|
---|
| 1474 |
|
---|
| 1475 | /* Because we're lazy, we'll jump back here to recalc 'off' and share the
|
---|
| 1476 | GCPhysRangePageWithOffset check. This is a little risky, so we use the
|
---|
| 1477 | 2nd register to check if we've looped more than once already.*/
|
---|
| 1478 | off = iemNativeEmitGprZero(pReNative, off, idxRegTmp2);
|
---|
| 1479 |
|
---|
| 1480 | uint32_t const offLabelRedoChecks = off;
|
---|
| 1481 |
|
---|
| 1482 | /* 1+2. Calculate 'off' first (into idxRegTmp). */
|
---|
| 1483 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
|
---|
| 1484 | if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
|
---|
| 1485 | {
|
---|
| 1486 | #ifdef RT_ARCH_ARM64
|
---|
| 1487 | uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
|
---|
| 1488 | pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
|
---|
| 1489 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1490 | #else
|
---|
| 1491 | off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
|
---|
| 1492 | off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
|
---|
| 1493 | #endif
|
---|
| 1494 | }
|
---|
| 1495 | else
|
---|
| 1496 | {
|
---|
| 1497 | #ifdef RT_ARCH_ARM64
|
---|
| 1498 | uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
|
---|
| 1499 | pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
|
---|
| 1500 | pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
|
---|
| 1501 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1502 | #else
|
---|
| 1503 | off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
|
---|
| 1504 | off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
|
---|
| 1505 | off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
|
---|
| 1506 | #endif
|
---|
| 1507 | }
|
---|
| 1508 |
|
---|
| 1509 | /* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal.
|
---|
| 1510 | Unlike iemNativeEmitBltInCheckPcAfterBranch we'll jump to the TLB loading if this fails. */
|
---|
| 1511 | off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
|
---|
| 1512 | uint32_t const offFixedJumpToTlbLoad = off;
|
---|
| 1513 | off = iemNativeEmitJaToFixed(pReNative, off, off /* (ASSUME ja rel8 suffices) */);
|
---|
| 1514 |
|
---|
| 1515 | /* 4a. Add iem.s.GCPhysInstrBuf to off ... */
|
---|
| 1516 | #ifdef RT_ARCH_AMD64
|
---|
| 1517 | uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
|
---|
| 1518 | pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
|
---|
| 1519 | pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
|
---|
| 1520 | off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
|
---|
| 1521 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1522 |
|
---|
| 1523 | #elif defined(RT_ARCH_ARM64)
|
---|
| 1524 |
|
---|
| 1525 | off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
|
---|
| 1526 | uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
|
---|
| 1527 | pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
|
---|
| 1528 | IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
|
---|
| 1529 |
|
---|
| 1530 | # ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
|
---|
| 1531 | off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /*fSetFlags*/);
|
---|
[102701] | 1532 | off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
|
---|
[102699] | 1533 | off = iemNativeEmitBrk(pReNative, off, 0x2005);
|
---|
| 1534 | # endif
|
---|
| 1535 | #else
|
---|
| 1536 | # error "Port me"
|
---|
| 1537 | #endif
|
---|
| 1538 |
|
---|
| 1539 | /* 4b. ... and compare with GCPhysRangePageWithOffset.
|
---|
| 1540 |
|
---|
| 1541 | Unlike iemNativeEmitBltInCheckPcAfterBranch we'll have to be more
|
---|
| 1542 | careful and avoid implicit temporary register usage here.
|
---|
| 1543 |
|
---|
| 1544 | Unlike the threaded version of this code, we do not obsolete TBs here to
|
---|
| 1545 | reduce the code size and because indirect calls may legally end at the
|
---|
| 1546 | same offset in two different pages depending on the program state. */
|
---|
| 1547 | /** @todo synch the threaded BODY_LOAD_TLB_AFTER_BRANCH version with this. */
|
---|
| 1548 | off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
|
---|
| 1549 | off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
|
---|
| 1550 | off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelCheckBranchMiss);
|
---|
| 1551 | uint32_t const offFixedJumpToEnd = off;
|
---|
| 1552 | off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
|
---|
| 1553 |
|
---|
| 1554 | /*
|
---|
[102856] | 1555 | * TlbLoad:
|
---|
| 1556 | *
|
---|
[102699] | 1557 | * First we try to go via the TLB.
|
---|
| 1558 | */
|
---|
| 1559 | iemNativeFixupFixedJump(pReNative, offFixedJumpToTlbLoad, off);
|
---|
| 1560 |
|
---|
| 1561 | /* Check that we haven't been here before. */
|
---|
| 1562 | off = iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, idxRegTmp2, false /*f64Bit*/, idxLabelCheckBranchMiss);
|
---|
| 1563 |
|
---|
[102856] | 1564 | /* Jump to the TLB lookup code. */
|
---|
| 1565 | uint32_t const idxLabelTlbLookup = !TlbState.fSkip
|
---|
| 1566 | ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
|
---|
| 1567 | : UINT32_MAX;
|
---|
| 1568 | //off = iemNativeEmitBrk(pReNative, off, 0x1234);
|
---|
| 1569 | if (!TlbState.fSkip)
|
---|
| 1570 | off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
|
---|
| 1571 |
|
---|
[102699] | 1572 | /*
|
---|
[102856] | 1573 | * TlbMiss:
|
---|
| 1574 | *
|
---|
| 1575 | * Call iemNativeHlpMemCodeNewPageTlbMiss to do the work.
|
---|
[102699] | 1576 | */
|
---|
[102856] | 1577 | uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
|
---|
| 1578 | RT_NOREF(idxLabelTlbMiss);
|
---|
[102699] | 1579 |
|
---|
[102846] | 1580 | /* Save variables in volatile registers. */
|
---|
[102856] | 1581 | uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() | RT_BIT_32(idxRegTmp) | RT_BIT_32(idxRegTmp2)
|
---|
| 1582 | | (idxRegDummy != UINT8_MAX ? RT_BIT_32(idxRegDummy) : 0);
|
---|
[102846] | 1583 | off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
|
---|
| 1584 |
|
---|
[102699] | 1585 | /* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
|
---|
| 1586 | off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
|
---|
| 1587 |
|
---|
| 1588 | /* Done setting up parameters, make the call. */
|
---|
| 1589 | off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMiss);
|
---|
| 1590 |
|
---|
[102846] | 1591 | /* Restore variables and guest shadow registers to volatile registers. */
|
---|
| 1592 | off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
|
---|
[102856] | 1593 | off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off,
|
---|
| 1594 | TlbState.getActiveRegsWithShadows()
|
---|
| 1595 | | RT_BIT_32(idxRegPc)
|
---|
| 1596 | | (idxRegCsBase != UINT8_MAX ? RT_BIT_32(idxRegCsBase) : 0));
|
---|
[102846] | 1597 |
|
---|
[102856] | 1598 | #ifdef IEMNATIVE_WITH_TLB_LOOKUP
|
---|
| 1599 | if (!TlbState.fSkip)
|
---|
| 1600 | {
|
---|
| 1601 | /* end of TlbMiss - Jump to the done label. */
|
---|
| 1602 | uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
|
---|
| 1603 | off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
|
---|
| 1604 |
|
---|
| 1605 | /*
|
---|
| 1606 | * TlbLookup:
|
---|
| 1607 | */
|
---|
| 1608 | off = iemNativeEmitTlbLookup<false, true>(pReNative, off, &TlbState, fIsFlat ? UINT8_MAX : X86_SREG_CS,
|
---|
| 1609 | 1 /*cbMem*/, 0 /*fAlignMask*/, IEM_ACCESS_TYPE_EXEC,
|
---|
| 1610 | idxLabelTlbLookup, idxLabelTlbMiss, idxRegDummy);
|
---|
| 1611 |
|
---|
| 1612 | # ifdef VBOX_WITH_STATISTICS
|
---|
| 1613 | off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
|
---|
| 1614 | RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPage));
|
---|
| 1615 | # endif
|
---|
| 1616 |
|
---|
| 1617 | /*
|
---|
| 1618 | * TlbDone:
|
---|
| 1619 | */
|
---|
| 1620 | iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
|
---|
| 1621 | TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /*idxVarGCPtrMem*/, true /*fIsCode*/);
|
---|
| 1622 | }
|
---|
| 1623 | #else
|
---|
| 1624 | RT_NOREF(idxLabelTlbMiss);
|
---|
| 1625 | #endif
|
---|
| 1626 |
|
---|
[102699] | 1627 | /* Jmp back to the start and redo the checks. */
|
---|
| 1628 | off = iemNativeEmitLoadGpr8Imm(pReNative, off, idxRegTmp2, 1); /* indicate that we've looped once already */
|
---|
| 1629 | off = iemNativeEmitJmpToFixed(pReNative, off, offLabelRedoChecks);
|
---|
| 1630 |
|
---|
[102856] | 1631 | /*
|
---|
| 1632 | * End:
|
---|
| 1633 | *
|
---|
| 1634 | * The end.
|
---|
| 1635 | */
|
---|
[102699] | 1636 | iemNativeFixupFixedJump(pReNative, offFixedJumpToEnd, off);
|
---|
| 1637 |
|
---|
[102856] | 1638 | if (!TlbState.fSkip)
|
---|
| 1639 | iemNativeRegFreeTmp(pReNative, idxRegDummy);
|
---|
| 1640 | else
|
---|
| 1641 | {
|
---|
| 1642 | iemNativeRegFreeTmp(pReNative, idxRegTmp2);
|
---|
| 1643 | iemNativeRegFreeTmp(pReNative, idxRegTmp);
|
---|
| 1644 | iemNativeRegFreeTmp(pReNative, idxRegPc);
|
---|
| 1645 | if (idxRegCsBase != UINT8_MAX)
|
---|
| 1646 | iemNativeRegFreeTmp(pReNative, idxRegCsBase);
|
---|
| 1647 | }
|
---|
[102699] | 1648 | return off;
|
---|
| 1649 | }
|
---|
| 1650 |
|
---|
| 1651 |
|
---|
[102593] | 1652 | #ifdef BODY_CHECK_CS_LIM
|
---|
| 1653 | /**
|
---|
| 1654 | * Built-in function that checks the EIP/IP + uParam0 is within CS.LIM,
|
---|
| 1655 | * raising a \#GP(0) if this isn't the case.
|
---|
| 1656 | */
|
---|
[102634] | 1657 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLim)
|
---|
[102593] | 1658 | {
|
---|
[103181] | 1659 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102603] | 1660 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1661 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 1662 | BODY_CHECK_CS_LIM(cbInstr);
|
---|
| 1663 | return off;
|
---|
| 1664 | }
|
---|
[103181] | 1665 |
|
---|
| 1666 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLim)
|
---|
| 1667 | {
|
---|
| 1668 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1669 | LIVENESS_CHECK_CS_LIM(pOutgoing);
|
---|
| 1670 | RT_NOREF(pCallEntry);
|
---|
| 1671 | }
|
---|
[102593] | 1672 | #endif
|
---|
| 1673 |
|
---|
| 1674 |
|
---|
| 1675 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_CS_LIM)
|
---|
| 1676 | /**
|
---|
| 1677 | * Built-in function for re-checking opcodes and CS.LIM after an instruction
|
---|
| 1678 | * that may have modified them.
|
---|
| 1679 | */
|
---|
[102634] | 1680 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodes)
|
---|
[102593] | 1681 | {
|
---|
[102603] | 1682 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1683 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1684 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1685 | uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
|
---|
[102603] | 1686 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1687 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 1688 | BODY_CHECK_CS_LIM(cbInstr);
|
---|
| 1689 | BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
|
---|
| 1690 | return off;
|
---|
| 1691 | }
|
---|
[103181] | 1692 |
|
---|
| 1693 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodes)
|
---|
| 1694 | {
|
---|
| 1695 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1696 | LIVENESS_CHECK_CS_LIM(pOutgoing);
|
---|
| 1697 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 1698 | RT_NOREF(pCallEntry);
|
---|
| 1699 | }
|
---|
[102593] | 1700 | #endif
|
---|
| 1701 |
|
---|
| 1702 |
|
---|
| 1703 | #if defined(BODY_CHECK_OPCODES)
|
---|
| 1704 | /**
|
---|
| 1705 | * Built-in function for re-checking opcodes after an instruction that may have
|
---|
| 1706 | * modified them.
|
---|
| 1707 | */
|
---|
[102634] | 1708 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodes)
|
---|
[102593] | 1709 | {
|
---|
| 1710 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1711 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1712 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1713 | uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
|
---|
[102603] | 1714 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1715 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 1716 | BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
|
---|
| 1717 | return off;
|
---|
| 1718 | }
|
---|
[103181] | 1719 |
|
---|
| 1720 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodes)
|
---|
| 1721 | {
|
---|
| 1722 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1723 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 1724 | RT_NOREF(pCallEntry);
|
---|
| 1725 | }
|
---|
[102593] | 1726 | #endif
|
---|
| 1727 |
|
---|
| 1728 |
|
---|
| 1729 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
|
---|
| 1730 | /**
|
---|
| 1731 | * Built-in function for re-checking opcodes and considering the need for CS.LIM
|
---|
| 1732 | * checking after an instruction that may have modified them.
|
---|
| 1733 | */
|
---|
[102634] | 1734 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesConsiderCsLim)
|
---|
[102593] | 1735 | {
|
---|
| 1736 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1737 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1738 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1739 | uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
|
---|
[102603] | 1740 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1741 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 1742 | BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
|
---|
| 1743 | BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
|
---|
| 1744 | return off;
|
---|
| 1745 | }
|
---|
[103181] | 1746 |
|
---|
| 1747 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesConsiderCsLim)
|
---|
| 1748 | {
|
---|
| 1749 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1750 | LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
|
---|
| 1751 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 1752 | RT_NOREF(pCallEntry);
|
---|
| 1753 | }
|
---|
[102593] | 1754 | #endif
|
---|
| 1755 |
|
---|
| 1756 |
|
---|
| 1757 | /*
|
---|
| 1758 | * Post-branching checkers.
|
---|
| 1759 | */
|
---|
| 1760 |
|
---|
| 1761 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
|
---|
| 1762 | /**
|
---|
| 1763 | * Built-in function for checking CS.LIM, checking the PC and checking opcodes
|
---|
| 1764 | * after conditional branching within the same page.
|
---|
| 1765 | *
|
---|
| 1766 | * @see iemThreadedFunc_BltIn_CheckPcAndOpcodes
|
---|
| 1767 | */
|
---|
[102634] | 1768 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndPcAndOpcodes)
|
---|
[102593] | 1769 | {
|
---|
| 1770 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1771 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1772 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1773 | uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
|
---|
| 1774 | //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
|
---|
[102603] | 1775 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1776 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 1777 | BODY_CHECK_CS_LIM(cbInstr);
|
---|
[102663] | 1778 | BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
|
---|
[102593] | 1779 | BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
|
---|
| 1780 | //LogFunc(("okay\n"));
|
---|
| 1781 | return off;
|
---|
| 1782 | }
|
---|
[103181] | 1783 |
|
---|
| 1784 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndPcAndOpcodes)
|
---|
| 1785 | {
|
---|
| 1786 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1787 | LIVENESS_CHECK_CS_LIM(pOutgoing);
|
---|
| 1788 | LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
|
---|
| 1789 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 1790 | RT_NOREF(pCallEntry);
|
---|
| 1791 | }
|
---|
[102593] | 1792 | #endif
|
---|
| 1793 |
|
---|
| 1794 |
|
---|
| 1795 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH)
|
---|
| 1796 | /**
|
---|
| 1797 | * Built-in function for checking the PC and checking opcodes after conditional
|
---|
| 1798 | * branching within the same page.
|
---|
| 1799 | *
|
---|
| 1800 | * @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
|
---|
| 1801 | */
|
---|
[102634] | 1802 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodes)
|
---|
[102593] | 1803 | {
|
---|
| 1804 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1805 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1806 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1807 | uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
|
---|
| 1808 | //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
|
---|
[102603] | 1809 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1810 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102663] | 1811 | BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
|
---|
[102593] | 1812 | BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
|
---|
| 1813 | //LogFunc(("okay\n"));
|
---|
| 1814 | return off;
|
---|
| 1815 | }
|
---|
[103181] | 1816 |
|
---|
| 1817 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodes)
|
---|
| 1818 | {
|
---|
| 1819 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1820 | LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
|
---|
| 1821 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 1822 | RT_NOREF(pCallEntry);
|
---|
| 1823 | }
|
---|
[102593] | 1824 | #endif
|
---|
| 1825 |
|
---|
| 1826 |
|
---|
| 1827 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
|
---|
| 1828 | /**
|
---|
| 1829 | * Built-in function for checking the PC and checking opcodes and considering
|
---|
| 1830 | * the need for CS.LIM checking after conditional branching within the same
|
---|
| 1831 | * page.
|
---|
| 1832 | *
|
---|
| 1833 | * @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
|
---|
| 1834 | */
|
---|
[102634] | 1835 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
|
---|
[102593] | 1836 | {
|
---|
| 1837 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1838 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1839 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1840 | uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
|
---|
| 1841 | //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
|
---|
[102603] | 1842 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1843 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 1844 | BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
|
---|
[102663] | 1845 | BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
|
---|
[102593] | 1846 | BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
|
---|
| 1847 | //LogFunc(("okay\n"));
|
---|
| 1848 | return off;
|
---|
| 1849 | }
|
---|
[103181] | 1850 |
|
---|
| 1851 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
|
---|
| 1852 | {
|
---|
| 1853 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1854 | LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
|
---|
| 1855 | LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
|
---|
| 1856 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 1857 | RT_NOREF(pCallEntry);
|
---|
| 1858 | }
|
---|
[102593] | 1859 | #endif
|
---|
| 1860 |
|
---|
| 1861 |
|
---|
| 1862 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
|
---|
| 1863 | /**
|
---|
| 1864 | * Built-in function for checking CS.LIM, loading TLB and checking opcodes when
|
---|
| 1865 | * transitioning to a different code page.
|
---|
| 1866 | *
|
---|
| 1867 | * The code page transition can either be natural over onto the next page (with
|
---|
| 1868 | * the instruction starting at page offset zero) or by means of branching.
|
---|
| 1869 | *
|
---|
| 1870 | * @see iemThreadedFunc_BltIn_CheckOpcodesLoadingTlb
|
---|
| 1871 | */
|
---|
[102634] | 1872 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
|
---|
[102593] | 1873 | {
|
---|
| 1874 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1875 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1876 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1877 | uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
|
---|
| 1878 | //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
|
---|
[102603] | 1879 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1880 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 1881 | BODY_CHECK_CS_LIM(cbInstr);
|
---|
[102699] | 1882 | Assert(offRange == 0);
|
---|
[102593] | 1883 | BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
|
---|
| 1884 | BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
|
---|
| 1885 | //LogFunc(("okay\n"));
|
---|
| 1886 | return off;
|
---|
| 1887 | }
|
---|
[103181] | 1888 |
|
---|
| 1889 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
|
---|
| 1890 | {
|
---|
| 1891 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1892 | LIVENESS_CHECK_CS_LIM(pOutgoing);
|
---|
| 1893 | LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
|
---|
| 1894 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 1895 | RT_NOREF(pCallEntry);
|
---|
| 1896 | }
|
---|
[102593] | 1897 | #endif
|
---|
| 1898 |
|
---|
| 1899 |
|
---|
| 1900 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH)
|
---|
| 1901 | /**
|
---|
| 1902 | * Built-in function for loading TLB and checking opcodes when transitioning to
|
---|
| 1903 | * a different code page.
|
---|
| 1904 | *
|
---|
| 1905 | * The code page transition can either be natural over onto the next page (with
|
---|
| 1906 | * the instruction starting at page offset zero) or by means of branching.
|
---|
| 1907 | *
|
---|
| 1908 | * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
|
---|
| 1909 | */
|
---|
[102634] | 1910 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlb)
|
---|
[102593] | 1911 | {
|
---|
| 1912 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1913 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1914 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1915 | uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
|
---|
| 1916 | //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
|
---|
[102603] | 1917 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1918 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102699] | 1919 | Assert(offRange == 0);
|
---|
[102593] | 1920 | BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
|
---|
| 1921 | BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
|
---|
| 1922 | //LogFunc(("okay\n"));
|
---|
| 1923 | return off;
|
---|
| 1924 | }
|
---|
[103181] | 1925 |
|
---|
| 1926 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlb)
|
---|
| 1927 | {
|
---|
| 1928 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1929 | LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
|
---|
| 1930 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 1931 | RT_NOREF(pCallEntry);
|
---|
| 1932 | }
|
---|
[102593] | 1933 | #endif
|
---|
| 1934 |
|
---|
| 1935 |
|
---|
| 1936 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
|
---|
| 1937 | /**
|
---|
| 1938 | * Built-in function for loading TLB and checking opcodes and considering the
|
---|
| 1939 | * need for CS.LIM checking when transitioning to a different code page.
|
---|
| 1940 | *
|
---|
| 1941 | * The code page transition can either be natural over onto the next page (with
|
---|
| 1942 | * the instruction starting at page offset zero) or by means of branching.
|
---|
| 1943 | *
|
---|
| 1944 | * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
|
---|
| 1945 | */
|
---|
[102634] | 1946 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
|
---|
[102593] | 1947 | {
|
---|
| 1948 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1949 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1950 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1951 | uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
|
---|
| 1952 | //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
|
---|
[102603] | 1953 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1954 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 1955 | BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
|
---|
[102699] | 1956 | Assert(offRange == 0);
|
---|
[102593] | 1957 | BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
|
---|
| 1958 | BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
|
---|
| 1959 | //LogFunc(("okay\n"));
|
---|
| 1960 | return off;
|
---|
| 1961 | }
|
---|
[103181] | 1962 |
|
---|
| 1963 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
|
---|
| 1964 | {
|
---|
| 1965 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 1966 | LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
|
---|
| 1967 | LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
|
---|
| 1968 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 1969 | RT_NOREF(pCallEntry);
|
---|
| 1970 | }
|
---|
[102593] | 1971 | #endif
|
---|
| 1972 |
|
---|
| 1973 |
|
---|
| 1974 |
|
---|
| 1975 | /*
|
---|
| 1976 | * Natural page crossing checkers.
|
---|
| 1977 | */
|
---|
| 1978 |
|
---|
| 1979 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
|
---|
| 1980 | /**
|
---|
| 1981 | * Built-in function for checking CS.LIM, loading TLB and checking opcodes on
|
---|
| 1982 | * both pages when transitioning to a different code page.
|
---|
| 1983 | *
|
---|
| 1984 | * This is used when the previous instruction requires revalidation of opcodes
|
---|
| 1985 | * bytes and the current instruction stries a page boundrary with opcode bytes
|
---|
| 1986 | * in both the old and new page.
|
---|
| 1987 | *
|
---|
| 1988 | * @see iemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb
|
---|
| 1989 | */
|
---|
[102634] | 1990 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
|
---|
[102593] | 1991 | {
|
---|
| 1992 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 1993 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 1994 | uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
|
---|
| 1995 | uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
|
---|
| 1996 | uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
|
---|
| 1997 | uint32_t const idxRange2 = idxRange1 + 1;
|
---|
[102603] | 1998 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 1999 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 2000 | BODY_CHECK_CS_LIM(cbInstr);
|
---|
| 2001 | BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
|
---|
| 2002 | BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
|
---|
| 2003 | BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
|
---|
| 2004 | return off;
|
---|
| 2005 | }
|
---|
[103181] | 2006 |
|
---|
| 2007 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
|
---|
| 2008 | {
|
---|
| 2009 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 2010 | LIVENESS_CHECK_CS_LIM(pOutgoing);
|
---|
| 2011 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 2012 | LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
|
---|
| 2013 | RT_NOREF(pCallEntry);
|
---|
| 2014 | }
|
---|
[102593] | 2015 | #endif
|
---|
| 2016 |
|
---|
| 2017 |
|
---|
| 2018 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
|
---|
| 2019 | /**
|
---|
| 2020 | * Built-in function for loading TLB and checking opcodes on both pages when
|
---|
| 2021 | * transitioning to a different code page.
|
---|
| 2022 | *
|
---|
| 2023 | * This is used when the previous instruction requires revalidation of opcodes
|
---|
| 2024 | * bytes and the current instruction stries a page boundrary with opcode bytes
|
---|
| 2025 | * in both the old and new page.
|
---|
| 2026 | *
|
---|
| 2027 | * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
|
---|
| 2028 | */
|
---|
[102634] | 2029 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
|
---|
[102593] | 2030 | {
|
---|
| 2031 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 2032 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 2033 | uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
|
---|
| 2034 | uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
|
---|
| 2035 | uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
|
---|
| 2036 | uint32_t const idxRange2 = idxRange1 + 1;
|
---|
[102603] | 2037 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 2038 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 2039 | BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
|
---|
| 2040 | BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
|
---|
| 2041 | BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
|
---|
| 2042 | return off;
|
---|
| 2043 | }
|
---|
[103181] | 2044 |
|
---|
| 2045 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
|
---|
| 2046 | {
|
---|
| 2047 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 2048 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 2049 | LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
|
---|
| 2050 | RT_NOREF(pCallEntry);
|
---|
| 2051 | }
|
---|
[102593] | 2052 | #endif
|
---|
| 2053 |
|
---|
| 2054 |
|
---|
| 2055 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
|
---|
| 2056 | /**
|
---|
| 2057 | * Built-in function for loading TLB and checking opcodes on both pages and
|
---|
| 2058 | * considering the need for CS.LIM checking when transitioning to a different
|
---|
| 2059 | * code page.
|
---|
| 2060 | *
|
---|
| 2061 | * This is used when the previous instruction requires revalidation of opcodes
|
---|
| 2062 | * bytes and the current instruction stries a page boundrary with opcode bytes
|
---|
| 2063 | * in both the old and new page.
|
---|
| 2064 | *
|
---|
| 2065 | * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
|
---|
| 2066 | */
|
---|
[102634] | 2067 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
|
---|
[102593] | 2068 | {
|
---|
| 2069 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 2070 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 2071 | uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
|
---|
| 2072 | uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
|
---|
| 2073 | uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
|
---|
| 2074 | uint32_t const idxRange2 = idxRange1 + 1;
|
---|
[102603] | 2075 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 2076 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 2077 | BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
|
---|
| 2078 | BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
|
---|
| 2079 | BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
|
---|
| 2080 | BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
|
---|
| 2081 | return off;
|
---|
| 2082 | }
|
---|
[103181] | 2083 |
|
---|
| 2084 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
|
---|
| 2085 | {
|
---|
| 2086 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 2087 | LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
|
---|
| 2088 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 2089 | LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
|
---|
| 2090 | RT_NOREF(pCallEntry);
|
---|
| 2091 | }
|
---|
[102593] | 2092 | #endif
|
---|
| 2093 |
|
---|
| 2094 |
|
---|
| 2095 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
|
---|
| 2096 | /**
|
---|
| 2097 | * Built-in function for checking CS.LIM, loading TLB and checking opcodes when
|
---|
| 2098 | * advancing naturally to a different code page.
|
---|
| 2099 | *
|
---|
| 2100 | * Only opcodes on the new page is checked.
|
---|
| 2101 | *
|
---|
| 2102 | * @see iemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb
|
---|
| 2103 | */
|
---|
[102634] | 2104 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
|
---|
[102593] | 2105 | {
|
---|
| 2106 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 2107 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 2108 | uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
|
---|
| 2109 | uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
|
---|
| 2110 | //uint32_t const offRange1 = (uint32_t)uParam2;
|
---|
| 2111 | uint32_t const idxRange2 = idxRange1 + 1;
|
---|
[102603] | 2112 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 2113 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 2114 | BODY_CHECK_CS_LIM(cbInstr);
|
---|
| 2115 | BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
|
---|
| 2116 | BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
|
---|
| 2117 | return off;
|
---|
| 2118 | }
|
---|
[103181] | 2119 |
|
---|
| 2120 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
|
---|
| 2121 | {
|
---|
| 2122 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 2123 | LIVENESS_CHECK_CS_LIM(pOutgoing);
|
---|
| 2124 | LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
|
---|
| 2125 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 2126 | RT_NOREF(pCallEntry);
|
---|
| 2127 | }
|
---|
[102593] | 2128 | #endif
|
---|
| 2129 |
|
---|
| 2130 |
|
---|
| 2131 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
|
---|
| 2132 | /**
|
---|
| 2133 | * Built-in function for loading TLB and checking opcodes when advancing
|
---|
| 2134 | * naturally to a different code page.
|
---|
| 2135 | *
|
---|
| 2136 | * Only opcodes on the new page is checked.
|
---|
| 2137 | *
|
---|
| 2138 | * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
|
---|
| 2139 | */
|
---|
[102634] | 2140 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
|
---|
[102593] | 2141 | {
|
---|
| 2142 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 2143 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 2144 | uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
|
---|
| 2145 | uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
|
---|
| 2146 | //uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
|
---|
| 2147 | uint32_t const idxRange2 = idxRange1 + 1;
|
---|
[102603] | 2148 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 2149 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 2150 | BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
|
---|
| 2151 | BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
|
---|
| 2152 | return off;
|
---|
| 2153 | }
|
---|
[103181] | 2154 |
|
---|
| 2155 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
|
---|
| 2156 | {
|
---|
| 2157 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 2158 | LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
|
---|
| 2159 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 2160 | RT_NOREF(pCallEntry);
|
---|
| 2161 | }
|
---|
[102593] | 2162 | #endif
|
---|
| 2163 |
|
---|
| 2164 |
|
---|
| 2165 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
|
---|
| 2166 | /**
|
---|
| 2167 | * Built-in function for loading TLB and checking opcodes and considering the
|
---|
| 2168 | * need for CS.LIM checking when advancing naturally to a different code page.
|
---|
| 2169 | *
|
---|
| 2170 | * Only opcodes on the new page is checked.
|
---|
| 2171 | *
|
---|
| 2172 | * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
|
---|
| 2173 | */
|
---|
[102634] | 2174 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
|
---|
[102593] | 2175 | {
|
---|
| 2176 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 2177 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 2178 | uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
|
---|
| 2179 | uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
|
---|
| 2180 | //uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
|
---|
| 2181 | uint32_t const idxRange2 = idxRange1 + 1;
|
---|
[102603] | 2182 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 2183 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 2184 | BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
|
---|
| 2185 | BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
|
---|
| 2186 | BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
|
---|
| 2187 | return off;
|
---|
| 2188 | }
|
---|
[103181] | 2189 |
|
---|
| 2190 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
|
---|
| 2191 | {
|
---|
| 2192 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 2193 | LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
|
---|
| 2194 | LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
|
---|
| 2195 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 2196 | RT_NOREF(pCallEntry);
|
---|
| 2197 | }
|
---|
[102593] | 2198 | #endif
|
---|
| 2199 |
|
---|
| 2200 |
|
---|
| 2201 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
|
---|
| 2202 | /**
|
---|
| 2203 | * Built-in function for checking CS.LIM, loading TLB and checking opcodes when
|
---|
| 2204 | * advancing naturally to a different code page with first instr at byte 0.
|
---|
| 2205 | *
|
---|
| 2206 | * @see iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb
|
---|
| 2207 | */
|
---|
[102634] | 2208 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
|
---|
[102593] | 2209 | {
|
---|
| 2210 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 2211 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 2212 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
[102603] | 2213 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 2214 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 2215 | BODY_CHECK_CS_LIM(cbInstr);
|
---|
| 2216 | BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
|
---|
| 2217 | //Assert(pVCpu->iem.s.offCurInstrStart == 0);
|
---|
| 2218 | BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
|
---|
| 2219 | return off;
|
---|
| 2220 | }
|
---|
[103181] | 2221 |
|
---|
| 2222 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
|
---|
| 2223 | {
|
---|
| 2224 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 2225 | LIVENESS_CHECK_CS_LIM(pOutgoing);
|
---|
| 2226 | LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
|
---|
| 2227 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 2228 | RT_NOREF(pCallEntry);
|
---|
| 2229 | }
|
---|
[102593] | 2230 | #endif
|
---|
| 2231 |
|
---|
| 2232 |
|
---|
| 2233 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
|
---|
| 2234 | /**
|
---|
| 2235 | * Built-in function for loading TLB and checking opcodes when advancing
|
---|
| 2236 | * naturally to a different code page with first instr at byte 0.
|
---|
| 2237 | *
|
---|
| 2238 | * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
|
---|
| 2239 | */
|
---|
[102634] | 2240 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
|
---|
[102593] | 2241 | {
|
---|
| 2242 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 2243 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 2244 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
[102603] | 2245 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 2246 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 2247 | BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
|
---|
| 2248 | //Assert(pVCpu->iem.s.offCurInstrStart == 0);
|
---|
| 2249 | BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
|
---|
| 2250 | return off;
|
---|
| 2251 | }
|
---|
[103181] | 2252 |
|
---|
| 2253 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
|
---|
| 2254 | {
|
---|
| 2255 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 2256 | LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
|
---|
| 2257 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 2258 | RT_NOREF(pCallEntry);
|
---|
| 2259 | }
|
---|
[102593] | 2260 | #endif
|
---|
| 2261 |
|
---|
| 2262 |
|
---|
| 2263 | #if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
|
---|
| 2264 | /**
|
---|
| 2265 | * Built-in function for loading TLB and checking opcodes and considering the
|
---|
| 2266 | * need for CS.LIM checking when advancing naturally to a different code page
|
---|
| 2267 | * with first instr at byte 0.
|
---|
| 2268 | *
|
---|
| 2269 | * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
|
---|
| 2270 | */
|
---|
[102634] | 2271 | IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
|
---|
[102593] | 2272 | {
|
---|
| 2273 | PCIEMTB const pTb = pReNative->pTbOrg;
|
---|
[103181] | 2274 | uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
|
---|
[102593] | 2275 | uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
|
---|
[102603] | 2276 | BODY_SET_CUR_INSTR();
|
---|
[102695] | 2277 | BODY_FLUSH_PENDING_WRITES();
|
---|
[102593] | 2278 | BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
|
---|
| 2279 | BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
|
---|
| 2280 | //Assert(pVCpu->iem.s.offCurInstrStart == 0);
|
---|
| 2281 | BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
|
---|
| 2282 | return off;
|
---|
| 2283 | }
|
---|
[103181] | 2284 |
|
---|
| 2285 | IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
|
---|
| 2286 | {
|
---|
| 2287 | IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
|
---|
| 2288 | LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
|
---|
| 2289 | LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
|
---|
| 2290 | LIVENESS_CHECK_OPCODES(pOutgoing);
|
---|
| 2291 | RT_NOREF(pCallEntry);
|
---|
| 2292 | }
|
---|
[102593] | 2293 | #endif
|
---|
| 2294 |
|
---|