VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp

Last change on this file was 104407, checked in by vboxsync, 4 weeks ago

VMM/IEM: Adjusted the TB exit statistics a bit more, adding a few new one, making more of the release stats that doesn't go into the TB, and organizing them to try avoid counting the same exit more than once. bugref:10376 bugref:10653
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 96.2 KB
Line 
1/* $Id: IEMAllN8veRecompBltIn.cpp 104407 2024-04-23 23:16:04Z vboxsync $ */
2/** @file
3 * IEM - Native Recompiler, Emitters for Built-In Threaded Functions.
4 */
5
6/*
7 * Copyright (C) 2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33//#define IEM_WITH_OPAQUE_DECODER_STATE - need offCurInstrStart access for iemNativeHlpMemCodeNewPageTlbMiss and friends.
34#define VMCPU_INCL_CPUM_GST_CTX
35#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
36#include <VBox/vmm/iem.h>
37#include <VBox/vmm/cpum.h>
38#include <VBox/vmm/dbgf.h>
39#include "IEMInternal.h"
40#include <VBox/vmm/vmcc.h>
41#include <VBox/log.h>
42#include <VBox/err.h>
43#include <VBox/param.h>
44#include <iprt/assert.h>
45#include <iprt/string.h>
46#if defined(RT_ARCH_AMD64)
47# include <iprt/x86.h>
48#elif defined(RT_ARCH_ARM64)
49# include <iprt/armv8.h>
50#endif
51
52
53#include "IEMInline.h"
54#include "IEMThreadedFunctions.h"
55#include "IEMN8veRecompiler.h"
56#include "IEMN8veRecompilerEmit.h"
57#include "IEMN8veRecompilerTlbLookup.h"
58
59
60
61/*********************************************************************************************************************************
62* TB Helper Functions *
63*********************************************************************************************************************************/
64#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_ARM64)
65DECLASM(void) iemNativeHlpAsmSafeWrapLogCpuState(void);
66#endif
67
68
69/**
70 * Used by TB code to deal with a TLB miss for a new page.
71 */
72IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCodeNewPageTlbMiss,(PVMCPUCC pVCpu))
73{
74 STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
75 pVCpu->iem.s.pbInstrBuf = NULL;
76 pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE;
77 pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
78 iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
79 if (pVCpu->iem.s.pbInstrBuf)
80 { /* likely */ }
81 else
82 {
83 IEM_DO_LONGJMP(pVCpu, VINF_IEM_REEXEC_BREAK);
84 }
85}
86
87
88/**
89 * Used by TB code to deal with a TLB miss for a new page.
90 */
91IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
92{
93 STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
94 pVCpu->iem.s.pbInstrBuf = NULL;
95 pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
96 pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
97 iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
98 return pVCpu->iem.s.pbInstrBuf ? pVCpu->iem.s.GCPhysInstrBuf : NIL_RTGCPHYS;
99}
100
101
102/*********************************************************************************************************************************
103* Builtin functions *
104*********************************************************************************************************************************/
105
106/**
107 * Built-in function that does nothing.
108 *
109 * Whether this is called or not can be controlled by the entry in the
110 * IEMThreadedGenerator.katBltIns table. This can be useful to determine
111 * whether why behaviour changes when enabling the LogCpuState builtins. I.e.
112 * whether it's the reduced call count in the TBs or the threaded calls flushing
113 * register state.
114 */
115IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Nop)
116{
117 RT_NOREF(pReNative, pCallEntry);
118 return off;
119}
120
121IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Nop)
122{
123 *pOutgoing = *pIncoming;
124 RT_NOREF(pCallEntry);
125}
126
127
128/**
129 * Emits for for LogCpuState.
130 *
131 * This shouldn't have any relevant impact on the recompiler state.
132 */
133IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_LogCpuState)
134{
135#ifdef RT_ARCH_AMD64
136 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
137 /* push rax */
138 pbCodeBuf[off++] = 0x50 + X86_GREG_xAX;
139 /* push imm32 */
140 pbCodeBuf[off++] = 0x68;
141 pbCodeBuf[off++] = RT_BYTE1(pCallEntry->auParams[0]);
142 pbCodeBuf[off++] = RT_BYTE2(pCallEntry->auParams[0]);
143 pbCodeBuf[off++] = RT_BYTE3(pCallEntry->auParams[0]);
144 pbCodeBuf[off++] = RT_BYTE4(pCallEntry->auParams[0]);
145 /* mov rax, iemNativeHlpAsmSafeWrapLogCpuState */
146 pbCodeBuf[off++] = X86_OP_REX_W;
147 pbCodeBuf[off++] = 0xb8 + X86_GREG_xAX;
148 *(uint64_t *)&pbCodeBuf[off] = (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState;
149 off += sizeof(uint64_t);
150 /* call rax */
151 pbCodeBuf[off++] = 0xff;
152 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
153 /* pop rax */
154 pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
155 /* pop rax */
156 pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
157#else
158 off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState);
159 RT_NOREF(pCallEntry);
160#endif
161
162 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
163 return off;
164}
165
166IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_LogCpuState)
167{
168 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
169 RT_NOREF(pCallEntry);
170}
171
172
173/**
174 * Built-in function that calls a C-implemention function taking zero arguments.
175 */
176IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_DeferToCImpl0)
177{
178 PFNIEMCIMPL0 const pfnCImpl = (PFNIEMCIMPL0)(uintptr_t)pCallEntry->auParams[0];
179 uint8_t const cbInstr = (uint8_t)pCallEntry->auParams[1];
180 uint64_t const fGstShwFlush = pCallEntry->auParams[2];
181 return iemNativeEmitCImplCall(pReNative, off, pCallEntry->idxInstr, fGstShwFlush, (uintptr_t)pfnCImpl, cbInstr, 0, 0, 0, 0);
182}
183
184IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_DeferToCImpl0)
185{
186 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
187 RT_NOREF(pCallEntry);
188}
189
190
191/**
192 * Flushes pending writes in preparation of raising an exception or aborting the TB.
193 */
194#define BODY_FLUSH_PENDING_WRITES() \
195 off = iemNativeRegFlushPendingWrites(pReNative, off);
196
197
198/**
199 * Built-in function that checks for pending interrupts that can be delivered or
200 * forced action flags.
201 *
202 * This triggers after the completion of an instruction, so EIP is already at
203 * the next instruction. If an IRQ or important FF is pending, this will return
204 * a non-zero status that stops TB execution.
205 */
206IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
207{
208 RT_NOREF(pCallEntry);
209
210 BODY_FLUSH_PENDING_WRITES();
211
212 /* It's too convenient to use iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet below
213 and I'm too lazy to create a 'Fixed' version of that one. */
214 uint32_t const idxLabelVmCheck = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckIrq,
215 UINT32_MAX, pReNative->uCheckIrqSeqNo++);
216
217 uint32_t const idxLabelReturnBreakFF = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnBreakFF);
218
219 /* Again, we need to load the extended EFLAGS before we actually need them
220 in case we jump. We couldn't use iemNativeRegAllocTmpForGuestReg if we
221 loaded them inside the check, as the shadow state would not be correct
222 when the code branches before the load. Ditto PC. */
223 uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
224 kIemNativeGstRegUse_ReadOnly);
225
226 uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ReadOnly);
227
228 uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
229
230 /*
231 * Start by checking the local forced actions of the EMT we're on for IRQs
232 * and other FFs that needs servicing.
233 */
234 /** @todo this isn't even close to the NMI and interrupt conditions in EM! */
235 /* Load FFs in to idxTmpReg and AND with all relevant flags. */
236 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
237 off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
238 VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
239 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
240 | VMCPU_FF_TLB_FLUSH
241 | VMCPU_FF_UNHALT ),
242 true /*fSetFlags*/);
243 /* If we end up with ZERO in idxTmpReg there is nothing to do.*/
244 uint32_t const offFixupJumpToVmCheck1 = off;
245 off = iemNativeEmitJzToFixed(pReNative, off, off /* ASSUME jz rel8 suffices */);
246
247 /* Some relevant FFs are set, but if's only APIC or/and PIC being set,
248 these may be supressed by EFLAGS.IF or CPUMIsInInterruptShadow. */
249 off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
250 ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC), true /*fSetFlags*/);
251 /* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
252 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreakFF);
253
254 /* So, it's only interrupt releated FFs and we need to see if IRQs are being
255 suppressed by the CPU or not. */
256 off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
257 off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
258 idxLabelReturnBreakFF);
259
260 /* We've got shadow flags set, so we must check that the PC they are valid
261 for matches our current PC value. */
262 /** @todo AMD64 can do this more efficiently w/o loading uRipInhibitInt into
263 * a register. */
264 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
265 off = iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, idxTmpReg, idxPcReg, idxLabelReturnBreakFF);
266
267 /*
268 * Now check the force flags of the VM.
269 */
270 iemNativeLabelDefine(pReNative, idxLabelVmCheck, off);
271 iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
272 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
273 off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
274 off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /*fSetFlags*/);
275 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreakFF);
276
277 /** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
278
279 /*
280 * We're good, no IRQs or FFs pending.
281 */
282 iemNativeRegFreeTmp(pReNative, idxTmpReg);
283 iemNativeRegFreeTmp(pReNative, idxEflReg);
284 iemNativeRegFreeTmp(pReNative, idxPcReg);
285
286 return off;
287}
288
289IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
290{
291 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
292 IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
293 RT_NOREF(pCallEntry);
294}
295
296
297/**
298 * Built-in function checks if IEMCPU::fExec has the expected value.
299 */
300IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckMode)
301{
302 uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
303 uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
304
305 off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
306 off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
307 off = iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
308 kIemNativeLabelType_ReturnBreak);
309 iemNativeRegFreeTmp(pReNative, idxTmpReg);
310
311 /* Maintain the recompiler fExec state. */
312 pReNative->fExec = fExpectedExec & IEMTB_F_IEM_F_MASK;
313 return off;
314}
315
316IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckMode)
317{
318 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
319 RT_NOREF(pCallEntry);
320}
321
322
323/**
324 * Sets idxTbCurInstr in preparation of raising an exception or aborting the TB.
325 */
326/** @todo Optimize this, so we don't set the same value more than once. Just
327 * needs some tracking. */
328#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
329# define BODY_SET_CUR_INSTR() \
330 off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, pCallEntry->idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr))
331#else
332# define BODY_SET_CUR_INSTR() ((void)0)
333#endif
334
335
336/**
337 * Macro that emits the 16/32-bit CS.LIM check.
338 */
339#define BODY_CHECK_CS_LIM(a_cbInstr) \
340 off = iemNativeEmitBltInCheckCsLim(pReNative, off, (a_cbInstr))
341
342#define LIVENESS_CHECK_CS_LIM(a_pOutgoing) \
343 IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS)
344
345DECL_FORCE_INLINE(uint32_t)
346iemNativeEmitBltInCheckCsLim(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
347{
348 Assert(cbInstr > 0);
349 Assert(cbInstr < 16);
350#ifdef VBOX_STRICT
351 off = iemNativeEmitMarker(pReNative, off, 0x80000001);
352#endif
353
354#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
355 Assert(pReNative->Core.offPc == 0);
356#endif
357
358 /*
359 * We need CS.LIM and RIP here. When cbInstr is larger than 1, we also need
360 * a temporary register for calculating the last address of the instruction.
361 *
362 * The calculation and comparisons are 32-bit. We ASSUME that the incoming
363 * RIP isn't totally invalid, i.e. that any jump/call/ret/iret instruction
364 * that last updated EIP here checked it already, and that we're therefore
365 * safe in the 32-bit wrap-around scenario to only check that the last byte
366 * is within CS.LIM. In the case of instruction-by-instruction advancing
367 * up to a EIP wrap-around, we know that CS.LIM is 4G-1 because the limit
368 * must be using 4KB granularity and the previous instruction was fine.
369 */
370 uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
371 kIemNativeGstRegUse_ReadOnly);
372 uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
373 kIemNativeGstRegUse_ReadOnly);
374#ifdef RT_ARCH_AMD64
375 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
376#elif defined(RT_ARCH_ARM64)
377 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
378#else
379# error "Port me"
380#endif
381
382 if (cbInstr != 1)
383 {
384 uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
385
386 /*
387 * 1. idxRegTmp = idxRegPc + cbInstr;
388 * 2. if idxRegTmp > idxRegCsLim then raise #GP(0).
389 */
390#ifdef RT_ARCH_AMD64
391 /* 1. lea tmp32, [Pc + cbInstr - 1] */
392 if (idxRegTmp >= 8 || idxRegPc >= 8)
393 pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) | (idxRegPc < 8 ? 0 : X86_OP_REX_B);
394 pbCodeBuf[off++] = 0x8d;
395 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, idxRegTmp & 7, idxRegPc & 7);
396 if ((idxRegPc & 7) == X86_GREG_xSP)
397 pbCodeBuf[off++] = X86_SIB_MAKE(idxRegPc & 7, 4 /*no index*/, 0);
398 pbCodeBuf[off++] = cbInstr - 1;
399
400 /* 2. cmp tmp32(r), CsLim(r/m). */
401 if (idxRegTmp >= 8 || idxRegCsLim >= 8)
402 pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) | (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
403 pbCodeBuf[off++] = 0x3b;
404 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegCsLim & 7);
405
406#elif defined(RT_ARCH_ARM64)
407 /* 1. add tmp32, Pc, #cbInstr-1 */
408 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, idxRegTmp, idxRegPc, cbInstr - 1, false /*f64Bit*/);
409 /* 2. cmp tmp32, CsLim */
410 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR, idxRegTmp, idxRegCsLim,
411 false /*f64Bit*/, true /*fSetFlags*/);
412
413#endif
414 iemNativeRegFreeTmp(pReNative, idxRegTmp);
415 }
416 else
417 {
418 /*
419 * Here we can skip step 1 and compare PC and CS.LIM directly.
420 */
421#ifdef RT_ARCH_AMD64
422 /* 2. cmp eip(r), CsLim(r/m). */
423 if (idxRegPc >= 8 || idxRegCsLim >= 8)
424 pbCodeBuf[off++] = (idxRegPc < 8 ? 0 : X86_OP_REX_R) | (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
425 pbCodeBuf[off++] = 0x3b;
426 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegPc & 7, idxRegCsLim & 7);
427
428#elif defined(RT_ARCH_ARM64)
429 /* 2. cmp Pc, CsLim */
430 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR, idxRegPc, idxRegCsLim,
431 false /*f64Bit*/, true /*fSetFlags*/);
432
433#endif
434 }
435
436 /* 3. Jump if greater. */
437 off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_RaiseGp0);
438
439 iemNativeRegFreeTmp(pReNative, idxRegCsLim);
440 iemNativeRegFreeTmp(pReNative, idxRegPc);
441 return off;
442}
443
444
445/**
446 * Macro that considers whether we need CS.LIM checking after a branch or
447 * crossing over to a new page.
448 */
449#define BODY_CONSIDER_CS_LIM_CHECKING(a_pTb, a_cbInstr) \
450 RT_NOREF(a_cbInstr); \
451 off = iemNativeEmitBltInConsiderLimChecking(pReNative, off)
452
453#define LIVENESS_CONSIDER_CS_LIM_CHECKING(a_pOutgoing) \
454 IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS); \
455 IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS)
456
457DECL_FORCE_INLINE(uint32_t)
458iemNativeEmitBltInConsiderLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off)
459{
460#ifdef VBOX_STRICT
461 off = iemNativeEmitMarker(pReNative, off, 0x80000002);
462#endif
463
464#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
465 Assert(pReNative->Core.offPc == 0);
466#endif
467
468 /*
469 * This check must match the ones in the iem in iemGetTbFlagsForCurrentPc
470 * exactly:
471 *
472 * int64_t const offFromLim = (int64_t)pVCpu->cpum.GstCtx.cs.u32Limit - (int64_t)pVCpu->cpum.GstCtx.eip;
473 * if (offFromLim >= X86_PAGE_SIZE + 16 - (int32_t)(pVCpu->cpum.GstCtx.cs.u64Base & GUEST_PAGE_OFFSET_MASK))
474 * return fRet;
475 * return fRet | IEMTB_F_CS_LIM_CHECKS;
476 *
477 *
478 * We need EIP, CS.LIM and CS.BASE here.
479 */
480
481 /* Calculate the offFromLim first: */
482 uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
483 kIemNativeGstRegUse_ReadOnly);
484 uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
485 kIemNativeGstRegUse_ReadOnly);
486 uint8_t const idxRegLeft = iemNativeRegAllocTmp(pReNative, &off);
487
488#ifdef RT_ARCH_ARM64
489 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
490 pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegLeft, idxRegCsLim, idxRegPc);
491 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
492#else
493 off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegLeft, idxRegCsLim);
494 off = iemNativeEmitSubTwoGprs(pReNative, off, idxRegLeft, idxRegPc);
495#endif
496
497 iemNativeRegFreeTmp(pReNative, idxRegCsLim);
498 iemNativeRegFreeTmp(pReNative, idxRegPc);
499
500 /* Calculate the threshold level (right side). */
501 uint8_t const idxRegCsBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
502 kIemNativeGstRegUse_ReadOnly);
503 uint8_t const idxRegRight = iemNativeRegAllocTmp(pReNative, &off);
504
505#ifdef RT_ARCH_ARM64
506 pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
507 Assert(Armv8A64ConvertImmRImmS2Mask32(11, 0) == GUEST_PAGE_OFFSET_MASK);
508 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegRight, idxRegCsBase, 11, 0, false /*f64Bit*/);
509 pu32CodeBuf[off++] = Armv8A64MkInstrNeg(idxRegRight);
510 pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
511 pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
512 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
513
514#else
515 off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegRight, GUEST_PAGE_OFFSET_MASK);
516 off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegRight, idxRegCsBase);
517 off = iemNativeEmitNegGpr(pReNative, off, idxRegRight);
518 off = iemNativeEmitAddGprImm(pReNative, off, idxRegRight, X86_PAGE_SIZE + 16);
519#endif
520
521 iemNativeRegFreeTmp(pReNative, idxRegCsBase);
522
523 /* Compare the two and jump out if we're too close to the limit. */
524 off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
525 off = iemNativeEmitJlToNewLabel(pReNative, off, kIemNativeLabelType_NeedCsLimChecking);
526
527 iemNativeRegFreeTmp(pReNative, idxRegRight);
528 iemNativeRegFreeTmp(pReNative, idxRegLeft);
529 return off;
530}
531
532
533
534/**
535 * Macro that implements opcode (re-)checking.
536 */
537#define BODY_CHECK_OPCODES(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
538 RT_NOREF(a_cbInstr); \
539 off = iemNativeEmitBltInCheckOpcodes(pReNative, off, (a_pTb), (a_idxRange), (a_offRange))
540
541#define LIVENESS_CHECK_OPCODES(a_pOutgoing) ((void)0)
542
543#if 0 /* debugging aid */
544bool g_fBpOnObsoletion = false;
545# define BP_ON_OBSOLETION g_fBpOnObsoletion
546#else
547# define BP_ON_OBSOLETION 0
548#endif
549
550DECL_FORCE_INLINE(uint32_t)
551iemNativeEmitBltInCheckOpcodes(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint16_t offRange)
552{
553 Assert(idxRange < pTb->cRanges && pTb->cRanges <= RT_ELEMENTS(pTb->aRanges));
554 Assert(offRange < pTb->aRanges[idxRange].cbOpcodes);
555#ifdef VBOX_STRICT
556 off = iemNativeEmitMarker(pReNative, off, 0x80000003);
557#endif
558
559 uint32_t const idxLabelObsoleteTb = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ObsoleteTb);
560
561 /*
562 * Where to start and how much to compare.
563 *
564 * Looking at the ranges produced when r160746 was running a DOS VM with TB
565 * logging, the ranges can be anything from 1 byte to at least 0x197 bytes,
566 * with the 6, 5, 4, 7, 8, 40, 3, 2, 9 and 10 being the top 10 in the sample.
567 *
568 * The top 10 for the early boot phase of a 64-bit debian 9.4 VM: 5, 9, 8,
569 * 12, 10, 11, 6, 13, 15 and 16. Max 0x359 bytes. Same revision as above.
570 */
571 uint16_t offPage = pTb->aRanges[idxRange].offPhysPage + offRange;
572 uint16_t cbLeft = pTb->aRanges[idxRange].cbOpcodes - offRange;
573 Assert(cbLeft > 0);
574 uint8_t const *pbOpcodes = &pTb->pabOpcodes[pTb->aRanges[idxRange].offOpcodes + offRange];
575 uint32_t offConsolidatedJump = UINT32_MAX;
576
577#ifdef RT_ARCH_AMD64
578 /* AMD64/x86 offers a bunch of options. Smaller stuff will can be
579 completely inlined, for larger we use REPE CMPS. */
580# define CHECK_OPCODES_CMP_IMMXX(a_idxReg, a_bOpcode) /* cost: 3 bytes */ do { \
581 pbCodeBuf[off++] = a_bOpcode; \
582 Assert(offPage < 127); \
583 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 7, a_idxReg); \
584 pbCodeBuf[off++] = RT_BYTE1(offPage); \
585 } while (0)
586
587# define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
588 if (offConsolidatedJump != UINT32_MAX) \
589 { \
590 int32_t const offDisp = (int32_t)offConsolidatedJump - (int32_t)(off + 2); \
591 Assert(offDisp >= -128); \
592 pbCodeBuf[off++] = 0x75; /* jnz near */ \
593 pbCodeBuf[off++] = (uint8_t)offDisp; \
594 } \
595 else \
596 { \
597 pbCodeBuf[off++] = 0x74; /* jz near +5 */ \
598 pbCodeBuf[off++] = 0x05 + BP_ON_OBSOLETION; \
599 offConsolidatedJump = off; \
600 if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
601 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */ \
602 iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_Rel32, -4); \
603 pbCodeBuf[off++] = 0x00; \
604 pbCodeBuf[off++] = 0x00; \
605 pbCodeBuf[off++] = 0x00; \
606 pbCodeBuf[off++] = 0x00; \
607 } \
608 } while (0)
609
610# define CHECK_OPCODES_CMP_IMM32(a_idxReg) /* cost: 3+4+2 = 9 */ do { \
611 CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
612 pbCodeBuf[off++] = *pbOpcodes++; \
613 pbCodeBuf[off++] = *pbOpcodes++; \
614 pbCodeBuf[off++] = *pbOpcodes++; \
615 pbCodeBuf[off++] = *pbOpcodes++; \
616 cbLeft -= 4; \
617 offPage += 4; \
618 CHECK_OPCODES_CMP_JMP(); \
619 } while (0)
620
621# define CHECK_OPCODES_CMP_IMM16(a_idxReg) /* cost: 1+3+2+2 = 8 */ do { \
622 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP; \
623 CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
624 pbCodeBuf[off++] = *pbOpcodes++; \
625 pbCodeBuf[off++] = *pbOpcodes++; \
626 cbLeft -= 2; \
627 offPage += 2; \
628 CHECK_OPCODES_CMP_JMP(); \
629 } while (0)
630
631# define CHECK_OPCODES_CMP_IMM8(a_idxReg) /* cost: 3+1+2 = 6 */ do { \
632 CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x80); \
633 pbCodeBuf[off++] = *pbOpcodes++; \
634 cbLeft -= 1; \
635 offPage += 1; \
636 CHECK_OPCODES_CMP_JMP(); \
637 } while (0)
638
639# define CHECK_OPCODES_CMPSX(a_bOpcode, a_cbToSubtract, a_bPrefix) /* cost: 2+2 = 4 */ do { \
640 if (a_bPrefix) \
641 pbCodeBuf[off++] = (a_bPrefix); \
642 pbCodeBuf[off++] = (a_bOpcode); \
643 CHECK_OPCODES_CMP_JMP(); \
644 cbLeft -= (a_cbToSubtract); \
645 } while (0)
646
647# define CHECK_OPCODES_ECX_IMM(a_uValue) /* cost: 5 */ do { \
648 pbCodeBuf[off++] = 0xb8 + X86_GREG_xCX; \
649 pbCodeBuf[off++] = RT_BYTE1(a_uValue); \
650 pbCodeBuf[off++] = RT_BYTE2(a_uValue); \
651 pbCodeBuf[off++] = RT_BYTE3(a_uValue); \
652 pbCodeBuf[off++] = RT_BYTE4(a_uValue); \
653 } while (0)
654
655 if (cbLeft <= 24)
656 {
657 uint8_t const idxRegTmp = iemNativeRegAllocTmpEx(pReNative, &off,
658 ( RT_BIT_32(X86_GREG_xAX)
659 | RT_BIT_32(X86_GREG_xCX)
660 | RT_BIT_32(X86_GREG_xDX)
661 | RT_BIT_32(X86_GREG_xBX)
662 | RT_BIT_32(X86_GREG_xSI)
663 | RT_BIT_32(X86_GREG_xDI))
664 & ~IEMNATIVE_REG_FIXED_MASK); /* pick reg not requiring rex prefix */
665 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf));
666 if (offPage >= 128 - cbLeft)
667 {
668 off = iemNativeEmitAddGprImm(pReNative, off, idxRegTmp, offPage & ~(uint16_t)3);
669 offPage &= 3;
670 }
671
672 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 87 */);
673
674 if (cbLeft > 8)
675 switch (offPage & 3)
676 {
677 case 0:
678 break;
679 case 1: /* cost: 6 + 8 = 14 */
680 CHECK_OPCODES_CMP_IMM8(idxRegTmp);
681 RT_FALL_THRU();
682 case 2: /* cost: 8 */
683 CHECK_OPCODES_CMP_IMM16(idxRegTmp);
684 break;
685 case 3: /* cost: 6 */
686 CHECK_OPCODES_CMP_IMM8(idxRegTmp);
687 break;
688 }
689
690 while (cbLeft >= 4)
691 CHECK_OPCODES_CMP_IMM32(idxRegTmp); /* max iteration: 24/4 = 6; --> cost: 6 * 9 = 54 */
692
693 if (cbLeft >= 2)
694 CHECK_OPCODES_CMP_IMM16(idxRegTmp); /* cost: 8 */
695 if (cbLeft)
696 CHECK_OPCODES_CMP_IMM8(idxRegTmp); /* cost: 6 */
697
698 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
699 iemNativeRegFreeTmp(pReNative, idxRegTmp);
700 }
701 else
702 {
703 /* RDI = &pbInstrBuf[offPage] */
704 uint8_t const idxRegDi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xDI));
705 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegDi, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
706 if (offPage != 0)
707 off = iemNativeEmitAddGprImm(pReNative, off, idxRegDi, offPage);
708
709 /* RSI = pbOpcodes */
710 uint8_t const idxRegSi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xSI));
711 off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegSi, (uintptr_t)pbOpcodes);
712
713 /* RCX = counts. */
714 uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
715
716 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /*= 35*/);
717
718 /** @todo profile and optimize this further. Maybe an idea to align by
719 * offPage if the two cannot be reconsidled. */
720 /* Align by the page offset, so that at least one of the accesses are naturally aligned. */
721 switch (offPage & 7) /* max cost: 10 */
722 {
723 case 0:
724 break;
725 case 1: /* cost: 3+4+3 = 10 */
726 CHECK_OPCODES_CMPSX(0xa6, 1, 0);
727 RT_FALL_THRU();
728 case 2: /* cost: 4+3 = 7 */
729 CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
730 CHECK_OPCODES_CMPSX(0xa7, 4, 0);
731 break;
732 case 3: /* cost: 3+3 = 6 */
733 CHECK_OPCODES_CMPSX(0xa6, 1, 0);
734 RT_FALL_THRU();
735 case 4: /* cost: 3 */
736 CHECK_OPCODES_CMPSX(0xa7, 4, 0);
737 break;
738 case 5: /* cost: 3+4 = 7 */
739 CHECK_OPCODES_CMPSX(0xa6, 1, 0);
740 RT_FALL_THRU();
741 case 6: /* cost: 4 */
742 CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
743 break;
744 case 7: /* cost: 3 */
745 CHECK_OPCODES_CMPSX(0xa6, 1, 0);
746 break;
747 }
748
749 /* Compare qwords: */
750 uint32_t const cQWords = cbLeft >> 3;
751 CHECK_OPCODES_ECX_IMM(cQWords); /* cost: 5 */
752
753 pbCodeBuf[off++] = X86_OP_PRF_REPZ; /* cost: 5 */
754 CHECK_OPCODES_CMPSX(0xa7, 0, X86_OP_REX_W);
755 cbLeft &= 7;
756
757 if (cbLeft & 4)
758 CHECK_OPCODES_CMPSX(0xa7, 4, 0); /* cost: 3 */
759 if (cbLeft & 2)
760 CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP); /* cost: 4 */
761 if (cbLeft & 1)
762 CHECK_OPCODES_CMPSX(0xa6, 1, 0); /* cost: 3 */
763
764 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
765 iemNativeRegFreeTmp(pReNative, idxRegCx);
766 iemNativeRegFreeTmp(pReNative, idxRegSi);
767 iemNativeRegFreeTmp(pReNative, idxRegDi);
768 }
769
770#elif defined(RT_ARCH_ARM64)
771 /* We need pbInstrBuf in a register, whatever we do. */
772 uint8_t const idxRegSrc1Ptr = iemNativeRegAllocTmp(pReNative, &off);
773 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegSrc1Ptr, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
774
775 /* We also need at least one more register for holding bytes & words we
776 load via pbInstrBuf. */
777 uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
778
779 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
780
781 /* One byte compare can be done with the opcode byte as an immediate. We'll
782 do this to uint16_t align src1. */
783 bool fPendingJmp = RT_BOOL(offPage & 1);
784 if (fPendingJmp)
785 {
786 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
787 pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, *pbOpcodes++, false /*f64Bit*/);
788 offPage += 1;
789 cbLeft -= 1;
790 }
791
792 if (cbLeft > 0)
793 {
794 /* We need a register for holding the opcode bytes we're comparing with,
795 as CCMP only has a 5-bit immediate form and thus cannot hold bytes. */
796 uint8_t const idxRegSrc2Val = iemNativeRegAllocTmp(pReNative, &off);
797
798 /* Word (uint32_t) aligning the src1 pointer is best done using a 16-bit constant load. */
799 if ((offPage & 3) && cbLeft >= 2)
800 {
801 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 2);
802 pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
803 if (fPendingJmp)
804 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
805 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
806 else
807 {
808 pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /*f64Bit*/);
809 fPendingJmp = true;
810 }
811 pbOpcodes += 2;
812 offPage += 2;
813 cbLeft -= 2;
814 }
815
816 /* DWord (uint64_t) aligning the src2 pointer. We use a 32-bit constant here for simplicitly. */
817 if ((offPage & 7) && cbLeft >= 4)
818 {
819 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 4);
820 off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
821 RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
822 if (fPendingJmp)
823 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
824 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
825 else
826 {
827 pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /*f64Bit*/);
828 fPendingJmp = true;
829 }
830 pbOpcodes += 4;
831 offPage += 4;
832 cbLeft -= 4;
833 }
834
835 /*
836 * If we've got 16 bytes or more left, switch to memcmp-style.
837 */
838 if (cbLeft >= 16)
839 {
840 /* We need a pointer to the copy of the original opcode bytes. */
841 uint8_t const idxRegSrc2Ptr = iemNativeRegAllocTmp(pReNative, &off);
842 off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Ptr, (uintptr_t)pbOpcodes);
843
844 /* If there are more than 32 bytes to compare we create a loop, for
845 which we'll need a loop register. */
846 if (cbLeft >= 64)
847 {
848 if (fPendingJmp)
849 {
850 iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
851 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
852 fPendingJmp = false;
853 }
854
855 uint8_t const idxRegLoop = iemNativeRegAllocTmp(pReNative, &off);
856 uint16_t const cLoops = cbLeft / 32;
857 cbLeft = cbLeft % 32;
858 pbOpcodes += cLoops * 32;
859 pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegLoop, cLoops);
860
861 if (offPage != 0) /** @todo optimize out this instruction. */
862 {
863 pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, offPage);
864 offPage = 0;
865 }
866
867 uint32_t const offLoopStart = off;
868 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 0);
869 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 0);
870 pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
871
872 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 1);
873 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 1);
874 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
875 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
876
877 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 2);
878 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 2);
879 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
880 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
881
882 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 3);
883 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 3);
884 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
885 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
886
887 iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
888 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
889
890 /* Advance and loop. */
891 pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, 0x20);
892 pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc2Ptr, idxRegSrc2Ptr, 0x20);
893 pu32CodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegLoop, idxRegLoop, 1, false /*f64Bit*/, true /*fSetFlags*/);
894 pu32CodeBuf[off] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, (int32_t)offLoopStart - (int32_t)off);
895 off++;
896
897 iemNativeRegFreeTmp(pReNative, idxRegLoop);
898 }
899
900 /* Deal with any remaining dwords (uint64_t). There can be up to
901 three if we looped and four if we didn't. */
902 uint32_t offSrc2 = 0;
903 while (cbLeft >= 8)
904 {
905 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val,
906 idxRegSrc1Ptr, offPage / 8);
907 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val,
908 idxRegSrc2Ptr, offSrc2 / 8);
909 if (fPendingJmp)
910 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
911 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
912 else
913 {
914 pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
915 fPendingJmp = true;
916 }
917 pbOpcodes += 8;
918 offPage += 8;
919 offSrc2 += 8;
920 cbLeft -= 8;
921 }
922
923 iemNativeRegFreeTmp(pReNative, idxRegSrc2Ptr);
924 /* max cost thus far: memcmp-loop=43 vs memcmp-no-loop=30 */
925 }
926 /*
927 * Otherwise, we compare with constants and merge with the general mop-up.
928 */
929 else
930 {
931 while (cbLeft >= 8)
932 {
933 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr,
934 offPage / 8);
935 off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Val,
936 RT_MAKE_U64_FROM_MSB_U8(pbOpcodes[7], pbOpcodes[6], pbOpcodes[5], pbOpcodes[4],
937 pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
938 if (fPendingJmp)
939 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
940 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, true /*f64Bit*/);
941 else
942 {
943 pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, true /*f64Bit*/);
944 fPendingJmp = true;
945 }
946 pbOpcodes += 8;
947 offPage += 8;
948 cbLeft -= 8;
949 }
950 /* max cost thus far: 21 */
951 }
952
953 /* Deal with any remaining bytes (7 or less). */
954 Assert(cbLeft < 8);
955 if (cbLeft >= 4)
956 {
957 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr,
958 offPage / 4);
959 off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
960 RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
961 if (fPendingJmp)
962 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
963 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
964 else
965 {
966 pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /*f64Bit*/);
967 fPendingJmp = true;
968 }
969 pbOpcodes += 4;
970 offPage += 4;
971 cbLeft -= 4;
972
973 }
974
975 if (cbLeft >= 2)
976 {
977 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr,
978 offPage / 2);
979 pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
980 if (fPendingJmp)
981 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
982 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
983 else
984 {
985 pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /*f64Bit*/);
986 fPendingJmp = true;
987 }
988 pbOpcodes += 2;
989 offPage += 2;
990 cbLeft -= 2;
991 }
992
993 if (cbLeft > 0)
994 {
995 Assert(cbLeft == 1);
996 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
997 if (fPendingJmp)
998 {
999 pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, pbOpcodes[0]);
1000 pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1001 ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /*f64Bit*/);
1002 }
1003 else
1004 {
1005 pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes[0], false /*f64Bit*/);
1006 fPendingJmp = true;
1007 }
1008 pbOpcodes += 1;
1009 offPage += 1;
1010 cbLeft -= 1;
1011 }
1012
1013 iemNativeRegFreeTmp(pReNative, idxRegSrc2Val);
1014 }
1015 Assert(cbLeft == 0);
1016
1017 /*
1018 * Finally, the branch on difference.
1019 */
1020 if (fPendingJmp)
1021 {
1022 iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
1023 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
1024 }
1025 RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump, idxLabelObsoleteTb);
1026
1027 /* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
1028 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1029 iemNativeRegFreeTmp(pReNative, idxRegSrc1Val);
1030 iemNativeRegFreeTmp(pReNative, idxRegSrc1Ptr);
1031
1032#else
1033# error "Port me"
1034#endif
1035 return off;
1036}
1037
1038
1039/** Duplicated in IEMAllThrdFuncsBltIn.cpp. */
1040DECL_FORCE_INLINE(RTGCPHYS) iemTbGetRangePhysPageAddr(PCIEMTB pTb, uint8_t idxRange)
1041{
1042 Assert(idxRange < RT_MIN(pTb->cRanges, RT_ELEMENTS(pTb->aRanges)));
1043 uint8_t const idxPage = pTb->aRanges[idxRange].idxPhysPage;
1044 Assert(idxPage <= RT_ELEMENTS(pTb->aGCPhysPages));
1045 if (idxPage == 0)
1046 return pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1047 Assert(!(pTb->aGCPhysPages[idxPage - 1] & GUEST_PAGE_OFFSET_MASK));
1048 return pTb->aGCPhysPages[idxPage - 1];
1049}
1050
1051
1052/**
1053 * Macro that implements PC check after a conditional branch.
1054 */
1055#define BODY_CHECK_PC_AFTER_BRANCH(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
1056 RT_NOREF(a_cbInstr); \
1057 off = iemNativeEmitBltInCheckPcAfterBranch(pReNative, off, a_pTb, a_idxRange, a_offRange)
1058
1059#define LIVENESS_CHECK_PC_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1060 if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1061 IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1062 else do { } while (0)
1063
1064DECL_FORCE_INLINE(uint32_t)
1065iemNativeEmitBltInCheckPcAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb,
1066 uint8_t idxRange, uint16_t offRange)
1067{
1068#ifdef VBOX_STRICT
1069 off = iemNativeEmitMarker(pReNative, off, 0x80000004);
1070#endif
1071
1072#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1073 Assert(pReNative->Core.offPc == 0);
1074#endif
1075
1076 /*
1077 * The GCPhysRangePageWithOffset value in the threaded function is a fixed
1078 * constant for us here.
1079 *
1080 * We can pretend that iem.s.cbInstrBufTotal is X86_PAGE_SIZE here, because
1081 * it serves no purpose as a CS.LIM, if that's needed we've just performed
1082 * it, and as long as we don't implement code TLB reload code here there is
1083 * no point in checking that the TLB data we're using is still valid.
1084 *
1085 * What we to do is.
1086 * 1. Calculate the FLAT PC (RIP + CS.BASE).
1087 * 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1088 * 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1089 * we're in the wrong spot and need to find a new TB.
1090 * 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1091 * GCPhysRangePageWithOffset constant mentioned above.
1092 *
1093 * The adding of CS.BASE to RIP can be skipped in the first step if we're
1094 * in 64-bit code or flat 32-bit.
1095 */
1096
1097 /* Allocate registers for step 1. Get the shadowed stuff before allocating
1098 the temp register, so we don't accidentally clobber something we'll be
1099 needing again immediately. This is why we get idxRegCsBase here. */
1100 uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1101 kIemNativeGstRegUse_ReadOnly);
1102 uint8_t const idxRegCsBase = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX
1103 : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1104 kIemNativeGstRegUse_ReadOnly);
1105
1106 uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
1107
1108#ifdef VBOX_STRICT
1109 /* Do assertions before idxRegTmp contains anything. */
1110 Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1111# ifdef RT_ARCH_AMD64
1112 {
1113 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1114 /* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1115 if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1116 {
1117 /* cmp r/m64, imm8 */
1118 pbCodeBuf[off++] = X86_OP_REX_W;
1119 pbCodeBuf[off++] = 0x83;
1120 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1121 pbCodeBuf[off++] = 0;
1122 /* je rel8 */
1123 pbCodeBuf[off++] = 0x74;
1124 pbCodeBuf[off++] = 1;
1125 /* int3 */
1126 pbCodeBuf[off++] = 0xcc;
1127
1128 }
1129
1130 /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1131 /* test r/m64, imm32 */
1132 pbCodeBuf[off++] = X86_OP_REX_W;
1133 pbCodeBuf[off++] = 0xf7;
1134 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1135 pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1136 pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1137 pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1138 pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1139 /* jz rel8 */
1140 pbCodeBuf[off++] = 0x74;
1141 pbCodeBuf[off++] = 1;
1142 /* int3 */
1143 pbCodeBuf[off++] = 0xcc;
1144 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1145 }
1146# else
1147
1148 /* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1149 if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1150 {
1151 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1152# ifdef RT_ARCH_ARM64
1153 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1154 pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 2, idxRegTmp);
1155 pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2004);
1156 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1157# else
1158# error "Port me!"
1159# endif
1160 }
1161# endif
1162
1163#endif /* VBOX_STRICT */
1164
1165 /* 1+2. Calculate 'off' first (into idxRegTmp). */
1166 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1167 if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1168 {
1169#ifdef RT_ARCH_ARM64
1170 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1171 pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1172 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1173#else
1174 off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1175 off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1176#endif
1177 }
1178 else
1179 {
1180#ifdef RT_ARCH_ARM64
1181 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1182 pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1183 pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1184 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1185#else
1186 off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1187 off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1188 off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1189#endif
1190 iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1191 }
1192 iemNativeRegFreeTmp(pReNative, idxRegPc);
1193
1194 /* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
1195 off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1196 off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
1197
1198 /* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
1199#ifdef RT_ARCH_AMD64
1200 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1201 pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
1202 pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1203 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1204 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1205
1206#elif defined(RT_ARCH_ARM64)
1207 uint8_t const idxRegTmp2 = iemNativeRegAllocTmp(pReNative, &off);
1208
1209 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1210 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1211 pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1212 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1213
1214# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1215 off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /*fSetFlags*/);
1216 off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1217 off = iemNativeEmitBrk(pReNative, off, 0x2005);
1218# endif
1219 iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1220#else
1221# error "Port me"
1222#endif
1223
1224 RTGCPHYS const GCPhysRangePageWithOffset = ( iemTbGetRangePhysPageAddr(pTb, idxRange)
1225 | pTb->aRanges[idxRange].offPhysPage)
1226 + offRange;
1227 off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
1228 kIemNativeLabelType_CheckBranchMiss);
1229
1230 iemNativeRegFreeTmp(pReNative, idxRegTmp);
1231 return off;
1232}
1233
1234
1235/**
1236 * Macro that implements TLB loading and updating pbInstrBuf updating for an
1237 * instruction crossing into a new page.
1238 *
1239 * This may long jump if we're raising a \#PF, \#GP or similar trouble.
1240 */
1241#define BODY_LOAD_TLB_FOR_NEW_PAGE(a_pTb, a_offInstr, a_idxRange, a_cbInstr) \
1242 RT_NOREF(a_cbInstr); \
1243 off = iemNativeEmitBltLoadTlbForNewPage(pReNative, off, pTb, a_idxRange, a_offInstr)
1244
1245#define LIVENESS_LOAD_TLB_FOR_NEW_PAGE(a_pOutgoing, a_pCallEntry) \
1246 if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1247 IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1248 else do { } while (0)
1249
1250DECL_FORCE_INLINE(uint32_t)
1251iemNativeEmitBltLoadTlbForNewPage(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint8_t offInstr)
1252{
1253#ifdef VBOX_STRICT
1254 off = iemNativeEmitMarker(pReNative, off, 0x80000005);
1255#endif
1256
1257 /*
1258 * Define labels and allocate the register for holding the GCPhys of the new page.
1259 */
1260 uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1261 uint32_t const idxRegGCPhys = iemNativeRegAllocTmp(pReNative, &off);
1262 IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
1263 uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1264 ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1265 : UINT32_MAX;
1266
1267 //off = iemNativeEmitBrk(pReNative, off, 0x1111);
1268
1269 /*
1270 * Jump to the TLB lookup code.
1271 */
1272 if (!TlbState.fSkip)
1273 off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1274
1275 /*
1276 * TlbMiss:
1277 *
1278 * Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
1279 */
1280 uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1281
1282 /* Save variables in volatile registers. */
1283 uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() | RT_BIT_32(idxRegGCPhys);
1284 off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1285
1286 /* IEMNATIVE_CALL_ARG1_GREG = offInstr */
1287 off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offInstr);
1288
1289 /* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1290 off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1291
1292 /* Done setting up parameters, make the call. */
1293 off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMissWithOff);
1294
1295 /* Move the result to the right register. */
1296 if (idxRegGCPhys != IEMNATIVE_CALL_RET_GREG)
1297 off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegGCPhys, IEMNATIVE_CALL_RET_GREG);
1298
1299 /* Restore variables and guest shadow registers to volatile registers. */
1300 off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1301 off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /*fCode*/));
1302
1303#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1304 if (!TlbState.fSkip)
1305 {
1306 /* end of TlbMiss - Jump to the done label. */
1307 uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1308 off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1309
1310 /*
1311 * TlbLookup:
1312 */
1313 off = iemNativeEmitTlbLookup<false>(pReNative, off, &TlbState,
1314 IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
1315 1 /*cbMem*/, 0 /*fAlignMask*/, IEM_ACCESS_TYPE_EXEC,
1316 idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
1317
1318# ifdef VBOX_WITH_STATISTICS
1319 off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1320 RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
1321# endif
1322
1323 /*
1324 * TlbDone:
1325 */
1326 iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1327 TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /*idxVarGCPtrMem*/, true /*fIsCode*/);
1328 }
1329#else
1330 RT_NOREF(idxLabelTlbMiss);
1331#endif
1332
1333 /*
1334 * Now check the physical address of the page matches the expected one.
1335 */
1336 RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
1337 off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegGCPhys, GCPhysNewPage,
1338 kIemNativeLabelType_ObsoleteTb);
1339
1340 iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
1341 return off;
1342}
1343
1344
1345/**
1346 * Macro that implements TLB loading and updating pbInstrBuf updating when
1347 * branching or when crossing a page on an instruction boundrary.
1348 *
1349 * This differs from BODY_LOAD_TLB_FOR_NEW_PAGE in that it will first check if
1350 * it is an inter-page branch and also check the page offset.
1351 *
1352 * This may long jump if we're raising a \#PF, \#GP or similar trouble.
1353 */
1354#define BODY_LOAD_TLB_AFTER_BRANCH(a_pTb, a_idxRange, a_cbInstr) \
1355 RT_NOREF(a_cbInstr); \
1356 off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
1357
1358#define LIVENESS_LOAD_TLB_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1359 if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1360 IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1361 else do { } while (0)
1362
1363DECL_FORCE_INLINE(uint32_t)
1364iemNativeEmitBltLoadTlbAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange)
1365{
1366#ifdef VBOX_STRICT
1367 off = iemNativeEmitMarker(pReNative, off, 0x80000006);
1368#endif
1369
1370 BODY_FLUSH_PENDING_WRITES();
1371
1372 /*
1373 * Define labels and allocate the register for holding the GCPhys of the new page.
1374 */
1375 uint32_t const idxLabelCheckBranchMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckBranchMiss);
1376 uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1377 RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
1378 | pTb->aRanges[idxRange].offPhysPage;
1379
1380 /*
1381 *
1382 * First check if RIP is within the current code.
1383 *
1384 * This is very similar to iemNativeEmitBltInCheckPcAfterBranch, the only
1385 * difference is what we do when stuff doesn't match up.
1386 *
1387 * What we to do is.
1388 * 1. Calculate the FLAT PC (RIP + CS.BASE).
1389 * 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1390 * 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1391 * we need to retranslate RIP via the TLB.
1392 * 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1393 * GCPhysRangePageWithOffset constant mentioned above.
1394 *
1395 * The adding of CS.BASE to RIP can be skipped in the first step if we're
1396 * in 64-bit code or flat 32-bit.
1397 *
1398 */
1399
1400 /* Allocate registers for step 1. Get the shadowed stuff before allocating
1401 the temp register, so we don't accidentally clobber something we'll be
1402 needing again immediately. This is why we get idxRegCsBase here.
1403 Update: We share registers with the TlbState, as the TLB code path has
1404 little in common with the rest of the code. */
1405 bool const fIsFlat = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec);
1406 IEMNATIVEEMITTLBSTATE const TlbState(pReNative, fIsFlat, &off);
1407 uint8_t const idxRegPc = !TlbState.fSkip ? TlbState.idxRegPtr
1408 : iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1409 kIemNativeGstRegUse_ReadOnly, true /*fNoVolatileRegs*/);
1410 uint8_t const idxRegCsBase = !TlbState.fSkip || fIsFlat ? TlbState.idxRegSegBase
1411 : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1412 kIemNativeGstRegUse_ReadOnly, true /*fNoVolatileRegs*/);
1413
1414 uint8_t const idxRegTmp = !TlbState.fSkip ? TlbState.idxReg1 : iemNativeRegAllocTmp(pReNative, &off);
1415 uint8_t const idxRegTmp2 = !TlbState.fSkip ? TlbState.idxReg2 : iemNativeRegAllocTmp(pReNative, &off);
1416 uint8_t const idxRegDummy = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
1417
1418#ifdef VBOX_STRICT
1419 /* Do assertions before idxRegTmp contains anything. */
1420 Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1421# ifdef RT_ARCH_AMD64
1422 {
1423 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1424 /* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1425 if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1426 {
1427 /* cmp r/m64, imm8 */
1428 pbCodeBuf[off++] = X86_OP_REX_W;
1429 pbCodeBuf[off++] = 0x83;
1430 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1431 pbCodeBuf[off++] = 0;
1432 /* je rel8 */
1433 pbCodeBuf[off++] = 0x74;
1434 pbCodeBuf[off++] = 1;
1435 /* int3 */
1436 pbCodeBuf[off++] = 0xcc;
1437
1438 }
1439
1440 /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1441 /* test r/m64, imm32 */
1442 pbCodeBuf[off++] = X86_OP_REX_W;
1443 pbCodeBuf[off++] = 0xf7;
1444 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1445 pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1446 pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1447 pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1448 pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1449 /* jz rel8 */
1450 pbCodeBuf[off++] = 0x74;
1451 pbCodeBuf[off++] = 1;
1452 /* int3 */
1453 pbCodeBuf[off++] = 0xcc;
1454 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1455 }
1456# else
1457
1458 /* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1459 if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1460 {
1461 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1462# ifdef RT_ARCH_ARM64
1463 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1464 pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 2, idxRegTmp);
1465 pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2006);
1466 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1467# else
1468# error "Port me!"
1469# endif
1470 }
1471# endif
1472
1473#endif /* VBOX_STRICT */
1474
1475 /* Because we're lazy, we'll jump back here to recalc 'off' and share the
1476 GCPhysRangePageWithOffset check. This is a little risky, so we use the
1477 2nd register to check if we've looped more than once already.*/
1478 off = iemNativeEmitGprZero(pReNative, off, idxRegTmp2);
1479
1480 uint32_t const offLabelRedoChecks = off;
1481
1482 /* 1+2. Calculate 'off' first (into idxRegTmp). */
1483 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1484 if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1485 {
1486#ifdef RT_ARCH_ARM64
1487 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1488 pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1489 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1490#else
1491 off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1492 off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1493#endif
1494 }
1495 else
1496 {
1497#ifdef RT_ARCH_ARM64
1498 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1499 pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1500 pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1501 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1502#else
1503 off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1504 off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1505 off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1506#endif
1507 }
1508
1509 /* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal.
1510 Unlike iemNativeEmitBltInCheckPcAfterBranch we'll jump to the TLB loading if this fails. */
1511 off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1512 uint32_t const offFixedJumpToTlbLoad = off;
1513 off = iemNativeEmitJaToFixed(pReNative, off, off /* (ASSUME ja rel8 suffices) */);
1514
1515 /* 4a. Add iem.s.GCPhysInstrBuf to off ... */
1516#ifdef RT_ARCH_AMD64
1517 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1518 pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
1519 pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1520 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1521 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1522
1523#elif defined(RT_ARCH_ARM64)
1524
1525 off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1526 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1527 pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1528 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1529
1530# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1531 off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /*fSetFlags*/);
1532 off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1533 off = iemNativeEmitBrk(pReNative, off, 0x2005);
1534# endif
1535#else
1536# error "Port me"
1537#endif
1538
1539 /* 4b. ... and compare with GCPhysRangePageWithOffset.
1540
1541 Unlike iemNativeEmitBltInCheckPcAfterBranch we'll have to be more
1542 careful and avoid implicit temporary register usage here.
1543
1544 Unlike the threaded version of this code, we do not obsolete TBs here to
1545 reduce the code size and because indirect calls may legally end at the
1546 same offset in two different pages depending on the program state. */
1547 /** @todo synch the threaded BODY_LOAD_TLB_AFTER_BRANCH version with this. */
1548 off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
1549 off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
1550 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelCheckBranchMiss);
1551 uint32_t const offFixedJumpToEnd = off;
1552 off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
1553
1554 /*
1555 * TlbLoad:
1556 *
1557 * First we try to go via the TLB.
1558 */
1559 iemNativeFixupFixedJump(pReNative, offFixedJumpToTlbLoad, off);
1560
1561 /* Check that we haven't been here before. */
1562 off = iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, idxRegTmp2, false /*f64Bit*/, idxLabelCheckBranchMiss);
1563
1564 /* Jump to the TLB lookup code. */
1565 uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1566 ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1567 : UINT32_MAX;
1568//off = iemNativeEmitBrk(pReNative, off, 0x1234);
1569 if (!TlbState.fSkip)
1570 off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1571
1572 /*
1573 * TlbMiss:
1574 *
1575 * Call iemNativeHlpMemCodeNewPageTlbMiss to do the work.
1576 */
1577 uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1578 RT_NOREF(idxLabelTlbMiss);
1579
1580 /* Save variables in volatile registers. */
1581 uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() | RT_BIT_32(idxRegTmp) | RT_BIT_32(idxRegTmp2)
1582 | (idxRegDummy != UINT8_MAX ? RT_BIT_32(idxRegDummy) : 0);
1583 off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1584
1585 /* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1586 off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1587
1588 /* Done setting up parameters, make the call. */
1589 off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMiss);
1590
1591 /* Restore variables and guest shadow registers to volatile registers. */
1592 off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1593 off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off,
1594 TlbState.getActiveRegsWithShadows()
1595 | RT_BIT_32(idxRegPc)
1596 | (idxRegCsBase != UINT8_MAX ? RT_BIT_32(idxRegCsBase) : 0));
1597
1598#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1599 if (!TlbState.fSkip)
1600 {
1601 /* end of TlbMiss - Jump to the done label. */
1602 uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1603 off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1604
1605 /*
1606 * TlbLookup:
1607 */
1608 off = iemNativeEmitTlbLookup<false, true>(pReNative, off, &TlbState, fIsFlat ? UINT8_MAX : X86_SREG_CS,
1609 1 /*cbMem*/, 0 /*fAlignMask*/, IEM_ACCESS_TYPE_EXEC,
1610 idxLabelTlbLookup, idxLabelTlbMiss, idxRegDummy);
1611
1612# ifdef VBOX_WITH_STATISTICS
1613 off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1614 RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPage));
1615# endif
1616
1617 /*
1618 * TlbDone:
1619 */
1620 iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1621 TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /*idxVarGCPtrMem*/, true /*fIsCode*/);
1622 }
1623#else
1624 RT_NOREF(idxLabelTlbMiss);
1625#endif
1626
1627 /* Jmp back to the start and redo the checks. */
1628 off = iemNativeEmitLoadGpr8Imm(pReNative, off, idxRegTmp2, 1); /* indicate that we've looped once already */
1629 off = iemNativeEmitJmpToFixed(pReNative, off, offLabelRedoChecks);
1630
1631 /*
1632 * End:
1633 *
1634 * The end.
1635 */
1636 iemNativeFixupFixedJump(pReNative, offFixedJumpToEnd, off);
1637
1638 if (!TlbState.fSkip)
1639 iemNativeRegFreeTmp(pReNative, idxRegDummy);
1640 else
1641 {
1642 iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1643 iemNativeRegFreeTmp(pReNative, idxRegTmp);
1644 iemNativeRegFreeTmp(pReNative, idxRegPc);
1645 if (idxRegCsBase != UINT8_MAX)
1646 iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1647 }
1648 return off;
1649}
1650
1651
1652#ifdef BODY_CHECK_CS_LIM
1653/**
1654 * Built-in function that checks the EIP/IP + uParam0 is within CS.LIM,
1655 * raising a \#GP(0) if this isn't the case.
1656 */
1657IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLim)
1658{
1659 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1660 BODY_SET_CUR_INSTR();
1661 BODY_FLUSH_PENDING_WRITES();
1662 BODY_CHECK_CS_LIM(cbInstr);
1663 return off;
1664}
1665
1666IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLim)
1667{
1668 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1669 LIVENESS_CHECK_CS_LIM(pOutgoing);
1670 RT_NOREF(pCallEntry);
1671}
1672#endif
1673
1674
1675#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_CS_LIM)
1676/**
1677 * Built-in function for re-checking opcodes and CS.LIM after an instruction
1678 * that may have modified them.
1679 */
1680IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodes)
1681{
1682 PCIEMTB const pTb = pReNative->pTbOrg;
1683 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1684 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1685 uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1686 BODY_SET_CUR_INSTR();
1687 BODY_FLUSH_PENDING_WRITES();
1688 BODY_CHECK_CS_LIM(cbInstr);
1689 BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1690 return off;
1691}
1692
1693IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodes)
1694{
1695 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1696 LIVENESS_CHECK_CS_LIM(pOutgoing);
1697 LIVENESS_CHECK_OPCODES(pOutgoing);
1698 RT_NOREF(pCallEntry);
1699}
1700#endif
1701
1702
1703#if defined(BODY_CHECK_OPCODES)
1704/**
1705 * Built-in function for re-checking opcodes after an instruction that may have
1706 * modified them.
1707 */
1708IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodes)
1709{
1710 PCIEMTB const pTb = pReNative->pTbOrg;
1711 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1712 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1713 uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1714 BODY_SET_CUR_INSTR();
1715 BODY_FLUSH_PENDING_WRITES();
1716 BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1717 return off;
1718}
1719
1720IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodes)
1721{
1722 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1723 LIVENESS_CHECK_OPCODES(pOutgoing);
1724 RT_NOREF(pCallEntry);
1725}
1726#endif
1727
1728
1729#if defined(BODY_CHECK_OPCODES) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1730/**
1731 * Built-in function for re-checking opcodes and considering the need for CS.LIM
1732 * checking after an instruction that may have modified them.
1733 */
1734IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesConsiderCsLim)
1735{
1736 PCIEMTB const pTb = pReNative->pTbOrg;
1737 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1738 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1739 uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1740 BODY_SET_CUR_INSTR();
1741 BODY_FLUSH_PENDING_WRITES();
1742 BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1743 BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1744 return off;
1745}
1746
1747IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesConsiderCsLim)
1748{
1749 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1750 LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1751 LIVENESS_CHECK_OPCODES(pOutgoing);
1752 RT_NOREF(pCallEntry);
1753}
1754#endif
1755
1756
1757/*
1758 * Post-branching checkers.
1759 */
1760
1761#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1762/**
1763 * Built-in function for checking CS.LIM, checking the PC and checking opcodes
1764 * after conditional branching within the same page.
1765 *
1766 * @see iemThreadedFunc_BltIn_CheckPcAndOpcodes
1767 */
1768IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1769{
1770 PCIEMTB const pTb = pReNative->pTbOrg;
1771 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1772 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1773 uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1774 //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1775 BODY_SET_CUR_INSTR();
1776 BODY_FLUSH_PENDING_WRITES();
1777 BODY_CHECK_CS_LIM(cbInstr);
1778 BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1779 BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1780 //LogFunc(("okay\n"));
1781 return off;
1782}
1783
1784IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1785{
1786 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1787 LIVENESS_CHECK_CS_LIM(pOutgoing);
1788 LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1789 LIVENESS_CHECK_OPCODES(pOutgoing);
1790 RT_NOREF(pCallEntry);
1791}
1792#endif
1793
1794
1795#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH)
1796/**
1797 * Built-in function for checking the PC and checking opcodes after conditional
1798 * branching within the same page.
1799 *
1800 * @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1801 */
1802IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodes)
1803{
1804 PCIEMTB const pTb = pReNative->pTbOrg;
1805 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1806 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1807 uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1808 //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1809 BODY_SET_CUR_INSTR();
1810 BODY_FLUSH_PENDING_WRITES();
1811 BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1812 BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1813 //LogFunc(("okay\n"));
1814 return off;
1815}
1816
1817IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodes)
1818{
1819 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1820 LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1821 LIVENESS_CHECK_OPCODES(pOutgoing);
1822 RT_NOREF(pCallEntry);
1823}
1824#endif
1825
1826
1827#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1828/**
1829 * Built-in function for checking the PC and checking opcodes and considering
1830 * the need for CS.LIM checking after conditional branching within the same
1831 * page.
1832 *
1833 * @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1834 */
1835IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1836{
1837 PCIEMTB const pTb = pReNative->pTbOrg;
1838 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1839 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1840 uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1841 //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1842 BODY_SET_CUR_INSTR();
1843 BODY_FLUSH_PENDING_WRITES();
1844 BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1845 BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1846 BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1847 //LogFunc(("okay\n"));
1848 return off;
1849}
1850
1851IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1852{
1853 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1854 LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1855 LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1856 LIVENESS_CHECK_OPCODES(pOutgoing);
1857 RT_NOREF(pCallEntry);
1858}
1859#endif
1860
1861
1862#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1863/**
1864 * Built-in function for checking CS.LIM, loading TLB and checking opcodes when
1865 * transitioning to a different code page.
1866 *
1867 * The code page transition can either be natural over onto the next page (with
1868 * the instruction starting at page offset zero) or by means of branching.
1869 *
1870 * @see iemThreadedFunc_BltIn_CheckOpcodesLoadingTlb
1871 */
1872IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1873{
1874 PCIEMTB const pTb = pReNative->pTbOrg;
1875 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1876 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1877 uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1878 //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1879 BODY_SET_CUR_INSTR();
1880 BODY_FLUSH_PENDING_WRITES();
1881 BODY_CHECK_CS_LIM(cbInstr);
1882 Assert(offRange == 0);
1883 BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1884 BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1885 //LogFunc(("okay\n"));
1886 return off;
1887}
1888
1889IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1890{
1891 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1892 LIVENESS_CHECK_CS_LIM(pOutgoing);
1893 LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1894 LIVENESS_CHECK_OPCODES(pOutgoing);
1895 RT_NOREF(pCallEntry);
1896}
1897#endif
1898
1899
1900#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH)
1901/**
1902 * Built-in function for loading TLB and checking opcodes when transitioning to
1903 * a different code page.
1904 *
1905 * The code page transition can either be natural over onto the next page (with
1906 * the instruction starting at page offset zero) or by means of branching.
1907 *
1908 * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1909 */
1910IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlb)
1911{
1912 PCIEMTB const pTb = pReNative->pTbOrg;
1913 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1914 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1915 uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1916 //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1917 BODY_SET_CUR_INSTR();
1918 BODY_FLUSH_PENDING_WRITES();
1919 Assert(offRange == 0);
1920 BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1921 BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1922 //LogFunc(("okay\n"));
1923 return off;
1924}
1925
1926IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlb)
1927{
1928 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1929 LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1930 LIVENESS_CHECK_OPCODES(pOutgoing);
1931 RT_NOREF(pCallEntry);
1932}
1933#endif
1934
1935
1936#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1937/**
1938 * Built-in function for loading TLB and checking opcodes and considering the
1939 * need for CS.LIM checking when transitioning to a different code page.
1940 *
1941 * The code page transition can either be natural over onto the next page (with
1942 * the instruction starting at page offset zero) or by means of branching.
1943 *
1944 * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1945 */
1946IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1947{
1948 PCIEMTB const pTb = pReNative->pTbOrg;
1949 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1950 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1951 uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1952 //LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1953 BODY_SET_CUR_INSTR();
1954 BODY_FLUSH_PENDING_WRITES();
1955 BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1956 Assert(offRange == 0);
1957 BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1958 BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1959 //LogFunc(("okay\n"));
1960 return off;
1961}
1962
1963IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1964{
1965 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1966 LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1967 LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1968 LIVENESS_CHECK_OPCODES(pOutgoing);
1969 RT_NOREF(pCallEntry);
1970}
1971#endif
1972
1973
1974
1975/*
1976 * Natural page crossing checkers.
1977 */
1978
1979#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
1980/**
1981 * Built-in function for checking CS.LIM, loading TLB and checking opcodes on
1982 * both pages when transitioning to a different code page.
1983 *
1984 * This is used when the previous instruction requires revalidation of opcodes
1985 * bytes and the current instruction stries a page boundrary with opcode bytes
1986 * in both the old and new page.
1987 *
1988 * @see iemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb
1989 */
1990IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
1991{
1992 PCIEMTB const pTb = pReNative->pTbOrg;
1993 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1994 uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
1995 uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
1996 uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
1997 uint32_t const idxRange2 = idxRange1 + 1;
1998 BODY_SET_CUR_INSTR();
1999 BODY_FLUSH_PENDING_WRITES();
2000 BODY_CHECK_CS_LIM(cbInstr);
2001 BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2002 BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2003 BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2004 return off;
2005}
2006
2007IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
2008{
2009 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2010 LIVENESS_CHECK_CS_LIM(pOutgoing);
2011 LIVENESS_CHECK_OPCODES(pOutgoing);
2012 LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2013 RT_NOREF(pCallEntry);
2014}
2015#endif
2016
2017
2018#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2019/**
2020 * Built-in function for loading TLB and checking opcodes on both pages when
2021 * transitioning to a different code page.
2022 *
2023 * This is used when the previous instruction requires revalidation of opcodes
2024 * bytes and the current instruction stries a page boundrary with opcode bytes
2025 * in both the old and new page.
2026 *
2027 * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2028 */
2029IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2030{
2031 PCIEMTB const pTb = pReNative->pTbOrg;
2032 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2033 uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2034 uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2035 uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2036 uint32_t const idxRange2 = idxRange1 + 1;
2037 BODY_SET_CUR_INSTR();
2038 BODY_FLUSH_PENDING_WRITES();
2039 BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2040 BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2041 BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2042 return off;
2043}
2044
2045IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2046{
2047 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2048 LIVENESS_CHECK_OPCODES(pOutgoing);
2049 LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2050 RT_NOREF(pCallEntry);
2051}
2052#endif
2053
2054
2055#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2056/**
2057 * Built-in function for loading TLB and checking opcodes on both pages and
2058 * considering the need for CS.LIM checking when transitioning to a different
2059 * code page.
2060 *
2061 * This is used when the previous instruction requires revalidation of opcodes
2062 * bytes and the current instruction stries a page boundrary with opcode bytes
2063 * in both the old and new page.
2064 *
2065 * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2066 */
2067IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2068{
2069 PCIEMTB const pTb = pReNative->pTbOrg;
2070 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2071 uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2072 uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2073 uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2074 uint32_t const idxRange2 = idxRange1 + 1;
2075 BODY_SET_CUR_INSTR();
2076 BODY_FLUSH_PENDING_WRITES();
2077 BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2078 BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2079 BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2080 BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2081 return off;
2082}
2083
2084IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2085{
2086 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2087 LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2088 LIVENESS_CHECK_OPCODES(pOutgoing);
2089 LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2090 RT_NOREF(pCallEntry);
2091}
2092#endif
2093
2094
2095#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2096/**
2097 * Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2098 * advancing naturally to a different code page.
2099 *
2100 * Only opcodes on the new page is checked.
2101 *
2102 * @see iemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb
2103 */
2104IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2105{
2106 PCIEMTB const pTb = pReNative->pTbOrg;
2107 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2108 uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2109 uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2110 //uint32_t const offRange1 = (uint32_t)uParam2;
2111 uint32_t const idxRange2 = idxRange1 + 1;
2112 BODY_SET_CUR_INSTR();
2113 BODY_FLUSH_PENDING_WRITES();
2114 BODY_CHECK_CS_LIM(cbInstr);
2115 BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2116 BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2117 return off;
2118}
2119
2120IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2121{
2122 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2123 LIVENESS_CHECK_CS_LIM(pOutgoing);
2124 LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2125 LIVENESS_CHECK_OPCODES(pOutgoing);
2126 RT_NOREF(pCallEntry);
2127}
2128#endif
2129
2130
2131#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2132/**
2133 * Built-in function for loading TLB and checking opcodes when advancing
2134 * naturally to a different code page.
2135 *
2136 * Only opcodes on the new page is checked.
2137 *
2138 * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2139 */
2140IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2141{
2142 PCIEMTB const pTb = pReNative->pTbOrg;
2143 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2144 uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2145 uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2146 //uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2147 uint32_t const idxRange2 = idxRange1 + 1;
2148 BODY_SET_CUR_INSTR();
2149 BODY_FLUSH_PENDING_WRITES();
2150 BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2151 BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2152 return off;
2153}
2154
2155IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2156{
2157 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2158 LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2159 LIVENESS_CHECK_OPCODES(pOutgoing);
2160 RT_NOREF(pCallEntry);
2161}
2162#endif
2163
2164
2165#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2166/**
2167 * Built-in function for loading TLB and checking opcodes and considering the
2168 * need for CS.LIM checking when advancing naturally to a different code page.
2169 *
2170 * Only opcodes on the new page is checked.
2171 *
2172 * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2173 */
2174IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2175{
2176 PCIEMTB const pTb = pReNative->pTbOrg;
2177 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2178 uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2179 uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2180 //uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2181 uint32_t const idxRange2 = idxRange1 + 1;
2182 BODY_SET_CUR_INSTR();
2183 BODY_FLUSH_PENDING_WRITES();
2184 BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2185 BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2186 BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2187 return off;
2188}
2189
2190IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2191{
2192 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2193 LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2194 LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2195 LIVENESS_CHECK_OPCODES(pOutgoing);
2196 RT_NOREF(pCallEntry);
2197}
2198#endif
2199
2200
2201#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2202/**
2203 * Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2204 * advancing naturally to a different code page with first instr at byte 0.
2205 *
2206 * @see iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb
2207 */
2208IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2209{
2210 PCIEMTB const pTb = pReNative->pTbOrg;
2211 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2212 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2213 BODY_SET_CUR_INSTR();
2214 BODY_FLUSH_PENDING_WRITES();
2215 BODY_CHECK_CS_LIM(cbInstr);
2216 BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2217 //Assert(pVCpu->iem.s.offCurInstrStart == 0);
2218 BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2219 return off;
2220}
2221
2222IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2223{
2224 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2225 LIVENESS_CHECK_CS_LIM(pOutgoing);
2226 LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2227 LIVENESS_CHECK_OPCODES(pOutgoing);
2228 RT_NOREF(pCallEntry);
2229}
2230#endif
2231
2232
2233#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2234/**
2235 * Built-in function for loading TLB and checking opcodes when advancing
2236 * naturally to a different code page with first instr at byte 0.
2237 *
2238 * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2239 */
2240IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2241{
2242 PCIEMTB const pTb = pReNative->pTbOrg;
2243 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2244 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2245 BODY_SET_CUR_INSTR();
2246 BODY_FLUSH_PENDING_WRITES();
2247 BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2248 //Assert(pVCpu->iem.s.offCurInstrStart == 0);
2249 BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2250 return off;
2251}
2252
2253IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2254{
2255 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2256 LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2257 LIVENESS_CHECK_OPCODES(pOutgoing);
2258 RT_NOREF(pCallEntry);
2259}
2260#endif
2261
2262
2263#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2264/**
2265 * Built-in function for loading TLB and checking opcodes and considering the
2266 * need for CS.LIM checking when advancing naturally to a different code page
2267 * with first instr at byte 0.
2268 *
2269 * @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2270 */
2271IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2272{
2273 PCIEMTB const pTb = pReNative->pTbOrg;
2274 uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2275 uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2276 BODY_SET_CUR_INSTR();
2277 BODY_FLUSH_PENDING_WRITES();
2278 BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2279 BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2280 //Assert(pVCpu->iem.s.offCurInstrStart == 0);
2281 BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2282 return off;
2283}
2284
2285IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2286{
2287 IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2288 LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2289 LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2290 LIVENESS_CHECK_OPCODES(pOutgoing);
2291 RT_NOREF(pCallEntry);
2292}
2293#endif
2294
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use