VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 103288

Last change on this file since 103288 was 103181, checked in by vboxsync, 16 months ago

VMM/IEM: Liveness analysis, part 1. bugref:10372
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 51.7 KB
Line 
1/* $Id: IEMN8veRecompiler.h 103181 2024-02-03 02:13:06Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @def IEMNATIVE_WITH_TB_DEBUG_INFO
41 * Enables generating internal debug info for better TB disassembly dumping. */
42#if defined(DEBUG) || defined(DOXYGEN_RUNNING)
43# define IEMNATIVE_WITH_TB_DEBUG_INFO
44#endif
45
46/** @def IEMNATIVE_WITH_LIVENESS_ANALYSIS
47 * Enables liveness analysis. */
48#if 1 || defined(DOXYGEN_RUNNING)
49# define IEMNATIVE_WITH_LIVENESS_ANALYSIS
50#endif
51
52#ifdef VBOX_WITH_STATISTICS
53/** Always count instructions for now. */
54# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
55#endif
56
57
58/** @name Stack Frame Layout
59 *
60 * @{ */
61/** The size of the area for stack variables and spills and stuff.
62 * @note This limit is duplicated in the python script(s). We add 0x40 for
63 * alignment padding. */
64#define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)
65/** Number of 64-bit variable slots (0x100 / 8 = 32. */
66#define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)
67AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
68
69#ifdef RT_ARCH_AMD64
70/** An stack alignment adjustment (between non-volatile register pushes and
71 * the stack variable area, so the latter better aligned). */
72# define IEMNATIVE_FRAME_ALIGN_SIZE 8
73
74/** Number of stack arguments slots for calls made from the frame. */
75# ifdef RT_OS_WINDOWS
76# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
77# else
78# define IEMNATIVE_FRAME_STACK_ARG_COUNT 2
79# endif
80/** Number of any shadow arguments (spill area) for calls we make. */
81# ifdef RT_OS_WINDOWS
82# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
83# else
84# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
85# endif
86
87/** Frame pointer (RBP) relative offset of the last push. */
88# ifdef RT_OS_WINDOWS
89# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
90# else
91# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
92# endif
93/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
94 * address for it). */
95# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
96/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
97# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
98/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
99# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
100# ifdef RT_OS_WINDOWS
101/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
102# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
103/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
104# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
105# endif
106
107# ifdef RT_OS_WINDOWS
108/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
109# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
110/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
111# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
112/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
113# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
114/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
115# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
116# endif
117
118#elif RT_ARCH_ARM64
119/** No alignment padding needed for arm64. */
120# define IEMNATIVE_FRAME_ALIGN_SIZE 0
121/** No stack argument slots, got 8 registers for arguments will suffice. */
122# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
123/** There are no argument spill area. */
124# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
125
126/** Number of saved registers at the top of our stack frame.
127 * This includes the return address and old frame pointer, so x19 thru x30. */
128# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
129/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
130# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
131
132/** Frame pointer (BP) relative offset of the last push. */
133# define IEMNATIVE_FP_OFF_LAST_PUSH (10 * -8)
134
135/** Frame pointer (BP) relative offset of the stack variable area (the lowest
136 * address for it). */
137# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
138
139#else
140# error "port me"
141#endif
142/** @} */
143
144
145/** @name Fixed Register Allocation(s)
146 * @{ */
147/** @def IEMNATIVE_REG_FIXED_PVMCPU
148 * The number of the register holding the pVCpu pointer. */
149/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
150 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
151 * @note This not available on AMD64, only ARM64. */
152/** @def IEMNATIVE_REG_FIXED_TMP0
153 * Dedicated temporary register.
154 * @todo replace this by a register allocator and content tracker. */
155/** @def IEMNATIVE_REG_FIXED_MASK
156 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
157 * architecture. */
158#if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)
159# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
160# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
161# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
162 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
163 | RT_BIT_32(X86_GREG_xSP) \
164 | RT_BIT_32(X86_GREG_xBP) )
165
166#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
167# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
168# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
169# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
170# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
171 | RT_BIT_32(ARMV8_A64_REG_LR) \
172 | RT_BIT_32(ARMV8_A64_REG_BP) \
173 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
174 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
175 | RT_BIT_32(ARMV8_A64_REG_X18) \
176 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
177
178#else
179# error "port me"
180#endif
181/** @} */
182
183/** @name Call related registers.
184 * @{ */
185/** @def IEMNATIVE_CALL_RET_GREG
186 * The return value register. */
187/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
188 * Number of arguments in registers. */
189/** @def IEMNATIVE_CALL_ARG0_GREG
190 * The general purpose register carrying argument \#0. */
191/** @def IEMNATIVE_CALL_ARG1_GREG
192 * The general purpose register carrying argument \#1. */
193/** @def IEMNATIVE_CALL_ARG2_GREG
194 * The general purpose register carrying argument \#2. */
195/** @def IEMNATIVE_CALL_ARG3_GREG
196 * The general purpose register carrying argument \#3. */
197/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
198 * Mask of registers the callee will not save and may trash. */
199#ifdef RT_ARCH_AMD64
200# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
201
202# ifdef RT_OS_WINDOWS
203# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
204# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
205# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
206# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
207# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
208# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
209 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
210 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
211 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) )
212# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
213 | RT_BIT_32(X86_GREG_xCX) \
214 | RT_BIT_32(X86_GREG_xDX) \
215 | RT_BIT_32(X86_GREG_x8) \
216 | RT_BIT_32(X86_GREG_x9) \
217 | RT_BIT_32(X86_GREG_x10) \
218 | RT_BIT_32(X86_GREG_x11) )
219# else
220# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
221# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
222# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
223# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
224# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
225# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
226# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
227# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(IEMNATIVE_CALL_ARG0_GREG) \
228 | RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) \
229 | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
230 | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
231 | RT_BIT_32(IEMNATIVE_CALL_ARG4_GREG) \
232 | RT_BIT_32(IEMNATIVE_CALL_ARG5_GREG) )
233# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
234 | RT_BIT_32(X86_GREG_xCX) \
235 | RT_BIT_32(X86_GREG_xDX) \
236 | RT_BIT_32(X86_GREG_xDI) \
237 | RT_BIT_32(X86_GREG_xSI) \
238 | RT_BIT_32(X86_GREG_x8) \
239 | RT_BIT_32(X86_GREG_x9) \
240 | RT_BIT_32(X86_GREG_x10) \
241 | RT_BIT_32(X86_GREG_x11) )
242# endif
243
244#elif defined(RT_ARCH_ARM64)
245# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
246# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
247# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
248# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
249# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
250# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
251# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
252# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
253# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
254# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
255# define IEMNATIVE_CALL_ARGS_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
256 | RT_BIT_32(ARMV8_A64_REG_X1) \
257 | RT_BIT_32(ARMV8_A64_REG_X2) \
258 | RT_BIT_32(ARMV8_A64_REG_X3) \
259 | RT_BIT_32(ARMV8_A64_REG_X4) \
260 | RT_BIT_32(ARMV8_A64_REG_X5) \
261 | RT_BIT_32(ARMV8_A64_REG_X6) \
262 | RT_BIT_32(ARMV8_A64_REG_X7) )
263# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
264 | RT_BIT_32(ARMV8_A64_REG_X1) \
265 | RT_BIT_32(ARMV8_A64_REG_X2) \
266 | RT_BIT_32(ARMV8_A64_REG_X3) \
267 | RT_BIT_32(ARMV8_A64_REG_X4) \
268 | RT_BIT_32(ARMV8_A64_REG_X5) \
269 | RT_BIT_32(ARMV8_A64_REG_X6) \
270 | RT_BIT_32(ARMV8_A64_REG_X7) \
271 | RT_BIT_32(ARMV8_A64_REG_X8) \
272 | RT_BIT_32(ARMV8_A64_REG_X9) \
273 | RT_BIT_32(ARMV8_A64_REG_X10) \
274 | RT_BIT_32(ARMV8_A64_REG_X11) \
275 | RT_BIT_32(ARMV8_A64_REG_X12) \
276 | RT_BIT_32(ARMV8_A64_REG_X13) \
277 | RT_BIT_32(ARMV8_A64_REG_X14) \
278 | RT_BIT_32(ARMV8_A64_REG_X15) \
279 | RT_BIT_32(ARMV8_A64_REG_X16) \
280 | RT_BIT_32(ARMV8_A64_REG_X17) )
281
282#endif
283
284/** This is the maximum argument count we'll ever be needing. */
285#if defined(RT_OS_WINDOWS) && defined(VBOXSTRICTRC_STRICT_ENABLED)
286# define IEMNATIVE_CALL_MAX_ARG_COUNT 8
287#else
288# define IEMNATIVE_CALL_MAX_ARG_COUNT 7
289#endif
290/** @} */
291
292
293/** @def IEMNATIVE_HST_GREG_COUNT
294 * Number of host general purpose registers we tracker. */
295/** @def IEMNATIVE_HST_GREG_MASK
296 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
297 * inverted register masks and such to get down to a correct set of regs. */
298#ifdef RT_ARCH_AMD64
299# define IEMNATIVE_HST_GREG_COUNT 16
300# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
301
302#elif defined(RT_ARCH_ARM64)
303# define IEMNATIVE_HST_GREG_COUNT 32
304# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
305#else
306# error "Port me!"
307#endif
308
309
310/** Native code generator label types. */
311typedef enum
312{
313 kIemNativeLabelType_Invalid = 0,
314 /* Labels w/o data, only once instance per TB: */
315 kIemNativeLabelType_Return,
316 kIemNativeLabelType_ReturnBreak,
317 kIemNativeLabelType_ReturnWithFlags,
318 kIemNativeLabelType_NonZeroRetOrPassUp,
319 kIemNativeLabelType_RaiseGp0,
320 kIemNativeLabelType_ObsoleteTb,
321 kIemNativeLabelType_NeedCsLimChecking,
322 kIemNativeLabelType_CheckBranchMiss,
323 /* Labels with data, potentially multiple instances per TB: */
324 kIemNativeLabelType_FirstWithMultipleInstances,
325 kIemNativeLabelType_If = kIemNativeLabelType_FirstWithMultipleInstances,
326 kIemNativeLabelType_Else,
327 kIemNativeLabelType_Endif,
328 kIemNativeLabelType_CheckIrq,
329 kIemNativeLabelType_TlbLookup,
330 kIemNativeLabelType_TlbMiss,
331 kIemNativeLabelType_TlbDone,
332 kIemNativeLabelType_End
333} IEMNATIVELABELTYPE;
334
335/** Native code generator label definition. */
336typedef struct IEMNATIVELABEL
337{
338 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
339 * the epilog. */
340 uint32_t off;
341 /** The type of label (IEMNATIVELABELTYPE). */
342 uint16_t enmType;
343 /** Additional label data, type specific. */
344 uint16_t uData;
345} IEMNATIVELABEL;
346/** Pointer to a label. */
347typedef IEMNATIVELABEL *PIEMNATIVELABEL;
348
349
350/** Native code generator fixup types. */
351typedef enum
352{
353 kIemNativeFixupType_Invalid = 0,
354#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
355 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
356 kIemNativeFixupType_Rel32,
357#elif defined(RT_ARCH_ARM64)
358 /** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */
359 kIemNativeFixupType_RelImm26At0,
360 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */
361 kIemNativeFixupType_RelImm19At5,
362 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */
363 kIemNativeFixupType_RelImm14At5,
364#endif
365 kIemNativeFixupType_End
366} IEMNATIVEFIXUPTYPE;
367
368/** Native code generator fixup. */
369typedef struct IEMNATIVEFIXUP
370{
371 /** Code offset of the fixup location. */
372 uint32_t off;
373 /** The IEMNATIVELABEL this is a fixup for. */
374 uint16_t idxLabel;
375 /** The fixup type (IEMNATIVEFIXUPTYPE). */
376 uint8_t enmType;
377 /** Addend or other data. */
378 int8_t offAddend;
379} IEMNATIVEFIXUP;
380/** Pointer to a native code generator fixup. */
381typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
382
383
384typedef union IEMLIVENESSPART1
385{
386 uint64_t bm64;
387 RT_GCC_EXTENSION struct
388 { /* bit no */
389 uint64_t bmGprs : 32; /**< 0x00 / 0: The 16 general purpose registers. */
390 uint64_t u2UnusedPc : 2; /**< 0x20 / 32: (PC in ) */
391 uint64_t u6Padding : 6; /**< 0x22 / 34: */
392 uint64_t bmSegBase : 12; /**< 0x28 / 40: */
393 uint64_t bmSegAttrib : 12; /**< 0x34 / 52: */
394 };
395} IEMLIVENESSPART1;
396AssertCompileSize(IEMLIVENESSPART1, 8);
397
398typedef union IEMLIVENESSPART2
399{
400 uint64_t bm64;
401 RT_GCC_EXTENSION struct
402 { /* bit no */
403 uint64_t bmSegLimit : 12; /**< 0x40 / 64: */
404 uint64_t bmSegSel : 12; /**< 0x4c / 76: */
405 uint64_t u2EflOther : 2; /**< 0x58 / 88: Other EFLAGS bits (~X86_EFL_STATUS_BITS & X86_EFL_LIVE_MASK). */
406 uint64_t u2EflCf : 2; /**< 0x5a / 90: Carry flag (X86_EFL_CF / 0). */
407 uint64_t u2EflPf : 2; /**< 0x5c / 92: Parity flag (X86_EFL_PF / 2). */
408 uint64_t u2EflAf : 2; /**< 0x5e / 94: Auxilary carry flag (X86_EFL_AF / 4). */
409 uint64_t u2EflZf : 2; /**< 0x60 / 96: Zero flag (X86_EFL_ZF / 6). */
410 uint64_t u2EflSf : 2; /**< 0x62 / 98: Signed flag (X86_EFL_SF / 7). */
411 uint64_t u2EflOf : 2; /**< 0x64 /100: Overflow flag (X86_EFL_OF / 12). */
412 uint64_t u24Unused : 24; /* 0x66 /102 -> 0x80/128 */
413 };
414} IEMLIVENESSPART2;
415AssertCompileSize(IEMLIVENESSPART2, 8);
416
417/**
418 * A liveness state entry.
419 *
420 * The first 128 bits runs parallel to kIemNativeGstReg_xxx for the most part.
421 * Once we add a SSE register shadowing, we'll add another 64-bit element for
422 * that.
423 */
424typedef union IEMLIVENESSENTRY
425{
426 uint64_t bm64[16 / 8];
427 uint16_t bm32[16 / 4];
428 uint16_t bm16[16 / 2];
429 uint8_t bm8[16 / 1];
430 RT_GCC_EXTENSION struct
431 {
432 IEMLIVENESSPART1 s1;
433 IEMLIVENESSPART2 s2;
434 };
435} IEMLIVENESSENTRY;
436AssertCompileSize(IEMLIVENESSENTRY, 16);
437/** Pointer to a liveness state entry. */
438typedef IEMLIVENESSENTRY *PIEMLIVENESSENTRY;
439/** Pointer to a const liveness state entry. */
440typedef IEMLIVENESSENTRY const *PCIEMLIVENESSENTRY;
441
442/** @name 64-bit value masks for IEMLIVENESSENTRY.
443 * @{ */ /* 0xzzzzyyyyxxxxwwww */
444#define IEMLIVENESSPART1_MASK UINT64_C(0xffffff00ffffffff)
445#define IEMLIVENESSPART2_MASK UINT64_C(0x0000003fffffffff)
446
447#define IEMLIVENESSPART1_XCPT_OR_CALL UINT64_C(0xaaaaaa00aaaaaaaa)
448#define IEMLIVENESSPART2_XCPT_OR_CALL UINT64_C(0x0000002aaaaaaaaa)
449
450#define IEMLIVENESSPART1_ALL_UNUSED UINT64_C(0x5555550055555555)
451#define IEMLIVENESSPART2_ALL_UNUSED UINT64_C(0x0000001555555555)
452
453#define IEMLIVENESSPART1_ALL_EFL_MASK UINT64_C(0x0000000000000000)
454#define IEMLIVENESSPART2_ALL_EFL_MASK UINT64_C(0x0000003fff000000)
455
456#define IEMLIVENESSPART1_ALL_EFL_INPUT IEMLIVENESSPART1_ALL_EFL_MASK
457#define IEMLIVENESSPART2_ALL_EFL_INPUT IEMLIVENESSPART2_ALL_EFL_MASK
458/** @} */
459
460
461/** @name The liveness state for a register.
462 *
463 * The state values have been picked to with state accumulation in mind (what
464 * the iemNativeLivenessFunc_xxxx functions does), as that is the most
465 * performance critical work done with the values.
466 *
467 * This is a compressed state that only requires 2 bits per register.
468 * When accumulating state, we'll be using three IEMLIVENESSENTRY copies:
469 * 1. the incoming state from the following call,
470 * 2. the outgoing state for this call,
471 * 3. mask of the entries set in the 2nd.
472 *
473 * The mask entry (3rd one above) will be used both when updating the outgoing
474 * state and when merging in incoming state for registers not touched by the
475 * current call.
476 *
477 * @{ */
478/** The register will be clobbered and the current value thrown away.
479 *
480 * When this is applied to the state (2) we'll simply be AND'ing it with the
481 * (old) mask (3) and adding the register to the mask. This way we'll
482 * preserve the high priority IEMLIVENESS_STATE_XCPT_OR_CALL and
483 * IEMLIVENESS_STATE_INPUT states. */
484#define IEMLIVENESS_STATE_CLOBBERED 0
485/** The register is unused in the remainder of the TB.
486 *
487 * This is an initial state and can not be set by any of the
488 * iemNativeLivenessFunc_xxxx callbacks. */
489#define IEMLIVENESS_STATE_UNUSED 1
490/** The register value is required in a potential call or exception.
491 *
492 * This means that the register value must be calculated and is best written to
493 * the state, but that any shadowing registers can be flushed thereafter as it's
494 * not used again. This state has lower priority than IEMLIVENESS_STATE_INPUT.
495 *
496 * It is typically applied across the board, but we preserve incoming
497 * IEMLIVENESS_STATE_INPUT values. This latter means we have to do some extra
498 * trickery to filter out IEMLIVENESS_STATE_UNUSED:
499 * 1. r0 = old & ~mask;
500 * 2. r0 = t1 & (t1 >> 1)'
501 * 3. state |= r0 | 0b10;
502 * 4. mask = ~0;
503 */
504#define IEMLIVENESS_STATE_XCPT_OR_CALL 2
505/** The register value is used as input.
506 *
507 * This means that the register value must be calculated and it is best to keep
508 * it in a register. It does not need to be writtent out as such. This is the
509 * highest priority state.
510 *
511 * Whether the call modifies the register or not isn't relevant to earlier
512 * calls, so that's not recorded.
513 *
514 * When applying this state we just or in the value in the outgoing state and
515 * mask. */
516#define IEMLIVENESS_STATE_INPUT 3
517/** Mask of the state bits. */
518#define IEMLIVENESS_STATE_MASK 3
519/** The number of bits per state. */
520#define IEMLIVENESS_STATE_BIT_COUNT 2
521/** @} */
522
523/** @name Liveness helpers for builtin functions and similar.
524 *
525 * These are not used by IEM_MC_BEGIN/END blocks, IEMAllN8veLiveness.cpp has its
526 * own set of manimulator macros for those.
527 *
528 * @{ */
529/** Initializing the outgoing state with a potnetial xcpt or call state.
530 * This only works when all changes will be IEMLIVENESS_STATE_INPUT. */
531#define IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(a_pOutgoing, a_pIncoming) \
532 do { \
533 uint64_t uTmp1 = (a_pIncoming)->s1.bm64; \
534 uTmp1 = uTmp1 & (uTmp1 >> 1); \
535 (a_pOutgoing)->s1.bm64 = uTmp1 | IEMLIVENESSPART1_XCPT_OR_CALL; \
536 \
537 uint64_t uTmp2 = (a_pIncoming)->s2.bm64; \
538 uTmp2 = uTmp2 & (uTmp1 >> 1); \
539 (a_pOutgoing)->s2.bm64 = uTmp2 | IEMLIVENESSPART2_XCPT_OR_CALL; \
540 } while (0)
541
542/** Adds a segment base register as input to the outgoing state. */
543#define IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, a_iSReg) \
544 (a_pOutgoing)->s1.bmSegBase |= (uint32_t)IEMLIVENESS_STATE_INPUT << ((a_iSReg) * IEMLIVENESS_STATE_BIT_COUNT)
545
546/** Adds a segment attribute register as input to the outgoing state. */
547#define IEM_LIVENESS_RAW_SEG_ATTRIB_INPUT(a_pOutgoing, a_iSReg) \
548 (a_pOutgoing)->s1.bmSegAttrib |= (uint32_t)IEMLIVENESS_STATE_INPUT << ((a_iSReg) * IEMLIVENESS_STATE_BIT_COUNT)
549
550/** Adds a segment limit register as input to the outgoing state. */
551#define IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, a_iSReg) \
552 (a_pOutgoing)->s2.bmSegLimit |= (uint32_t)IEMLIVENESS_STATE_INPUT << ((a_iSReg) * IEMLIVENESS_STATE_BIT_COUNT)
553/** @} */
554
555/**
556 * Guest registers that can be shadowed in GPRs.
557 *
558 * This runs parallel to the first 128-bits of liveness state. To avoid having
559 * the SegLimitXxxx range cross from the 1st 64-bit word to the 2nd,
560 * we've inserted some padding. The EFlags must be placed last, as the liveness
561 * state tracks it as 7 subcomponents and we don't want to waste space here.
562 */
563typedef enum IEMNATIVEGSTREG : uint8_t
564{
565 kIemNativeGstReg_GprFirst = 0,
566 kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,
567 kIemNativeGstReg_Pc,
568 kIemNativeGstReg_LivenessPadding17,
569 kIemNativeGstReg_LivenessPadding18,
570 kIemNativeGstReg_LivenessPadding19,
571 kIemNativeGstReg_SegBaseFirst,
572 kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,
573 kIemNativeGstReg_SegAttribFirst,
574 kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
575 kIemNativeGstReg_SegLimitFirst,
576 kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,
577 kIemNativeGstReg_SegSelFirst,
578 kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,
579 kIemNativeGstReg_EFlags, /**< 32-bit, includes internal flags - last! */
580 kIemNativeGstReg_End
581} IEMNATIVEGSTREG;
582AssertCompile((int)kIemNativeGstReg_SegLimitFirst == 32);
583
584/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
585 * @{ */
586#define IEMNATIVEGSTREG_GPR(a_iGpr) ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + (a_iGpr) ))
587#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst + (a_iSegReg) ))
588#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst + (a_iSegReg) ))
589#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
590#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg) ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
591/** @} */
592
593/**
594 * Intended use statement for iemNativeRegAllocTmpForGuestReg().
595 */
596typedef enum IEMNATIVEGSTREGUSE
597{
598 /** The usage is read-only, the register holding the guest register
599 * shadow copy will not be modified by the caller. */
600 kIemNativeGstRegUse_ReadOnly = 0,
601 /** The caller will update the guest register (think: PC += cbInstr).
602 * The guest shadow copy will follow the returned register. */
603 kIemNativeGstRegUse_ForUpdate,
604 /** The call will put an entirely new value in the guest register, so
605 * if new register is allocate it will be returned uninitialized. */
606 kIemNativeGstRegUse_ForFullWrite,
607 /** The caller will use the guest register value as input in a calculation
608 * and the host register will be modified.
609 * This means that the returned host register will not be marked as a shadow
610 * copy of the guest register. */
611 kIemNativeGstRegUse_Calculation
612} IEMNATIVEGSTREGUSE;
613
614/**
615 * Guest registers (classes) that can be referenced.
616 */
617typedef enum IEMNATIVEGSTREGREF : uint8_t
618{
619 kIemNativeGstRegRef_Invalid = 0,
620 kIemNativeGstRegRef_Gpr,
621 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
622 kIemNativeGstRegRef_EFlags,
623 kIemNativeGstRegRef_MxCsr,
624 kIemNativeGstRegRef_FpuReg,
625 kIemNativeGstRegRef_MReg,
626 kIemNativeGstRegRef_XReg,
627 //kIemNativeGstRegRef_YReg, - doesn't work.
628 kIemNativeGstRegRef_End
629} IEMNATIVEGSTREGREF;
630
631
632/** Variable kinds. */
633typedef enum IEMNATIVEVARKIND : uint8_t
634{
635 /** Customary invalid zero value. */
636 kIemNativeVarKind_Invalid = 0,
637 /** This is either in a register or on the stack. */
638 kIemNativeVarKind_Stack,
639 /** Immediate value - loaded into register when needed, or can live on the
640 * stack if referenced (in theory). */
641 kIemNativeVarKind_Immediate,
642 /** Variable reference - loaded into register when needed, never stack. */
643 kIemNativeVarKind_VarRef,
644 /** Guest register reference - loaded into register when needed, never stack. */
645 kIemNativeVarKind_GstRegRef,
646 /** End of valid values. */
647 kIemNativeVarKind_End
648} IEMNATIVEVARKIND;
649
650
651/** Variable or argument. */
652typedef struct IEMNATIVEVAR
653{
654 /** The kind of variable. */
655 IEMNATIVEVARKIND enmKind;
656 /** The variable size in bytes. */
657 uint8_t cbVar;
658 /** The first stack slot (uint64_t), except for immediate and references
659 * where it usually is UINT8_MAX. This is allocated lazily, so if a variable
660 * has a stack slot it has been initialized and has a value. Unused variables
661 * has neither a stack slot nor a host register assignment. */
662 uint8_t idxStackSlot;
663 /** The host register allocated for the variable, UINT8_MAX if not. */
664 uint8_t idxReg;
665 /** The argument number if argument, UINT8_MAX if regular variable. */
666 uint8_t uArgNo;
667 /** If referenced, the index of the variable referencing this one, otherwise
668 * UINT8_MAX. A referenced variable must only be placed on the stack and
669 * must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
670 uint8_t idxReferrerVar;
671 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
672 * @todo not sure what this really is for... */
673 IEMNATIVEGSTREG enmGstReg;
674 /** Set if the registered is currently used exclusively, false if the
675 * variable is idle and the register can be grabbed. */
676 bool fRegAcquired;
677
678 union
679 {
680 /** kIemNativeVarKind_Immediate: The immediate value. */
681 uint64_t uValue;
682 /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
683 uint8_t idxRefVar;
684 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
685 struct
686 {
687 /** The class of register. */
688 IEMNATIVEGSTREGREF enmClass;
689 /** Index within the class. */
690 uint8_t idx;
691 } GstRegRef;
692 } u;
693} IEMNATIVEVAR;
694
695/** What is being kept in a host register. */
696typedef enum IEMNATIVEWHAT : uint8_t
697{
698 /** The traditional invalid zero value. */
699 kIemNativeWhat_Invalid = 0,
700 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
701 kIemNativeWhat_Var,
702 /** Temporary register, this is typically freed when a MC completes. */
703 kIemNativeWhat_Tmp,
704 /** Call argument w/o a variable mapping. This is free (via
705 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
706 kIemNativeWhat_Arg,
707 /** Return status code.
708 * @todo not sure if we need this... */
709 kIemNativeWhat_rc,
710 /** The fixed pVCpu (PVMCPUCC) register.
711 * @todo consider offsetting this on amd64 to use negative offsets to access
712 * more members using 8-byte disp. */
713 kIemNativeWhat_pVCpuFixed,
714 /** The fixed pCtx (PCPUMCTX) register.
715 * @todo consider offsetting this on amd64 to use negative offsets to access
716 * more members using 8-byte disp. */
717 kIemNativeWhat_pCtxFixed,
718 /** Fixed temporary register. */
719 kIemNativeWhat_FixedTmp,
720 /** Register reserved by the CPU or OS architecture. */
721 kIemNativeWhat_FixedReserved,
722 /** End of valid values. */
723 kIemNativeWhat_End
724} IEMNATIVEWHAT;
725
726/**
727 * Host general register entry.
728 *
729 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
730 *
731 * @todo Track immediate values in host registers similarlly to how we track the
732 * guest register shadow copies. For it to be real helpful, though,
733 * we probably need to know which will be reused and put them into
734 * non-volatile registers, otherwise it's going to be more or less
735 * restricted to an instruction or two.
736 */
737typedef struct IEMNATIVEHSTREG
738{
739 /** Set of guest registers this one shadows.
740 *
741 * Using a bitmap here so we can designate the same host register as a copy
742 * for more than one guest register. This is expected to be useful in
743 * situations where one value is copied to several registers in a sequence.
744 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
745 * sequence we'd want to let this register follow to be a copy of and there
746 * will always be places where we'd be picking the wrong one.
747 */
748 uint64_t fGstRegShadows;
749 /** What is being kept in this register. */
750 IEMNATIVEWHAT enmWhat;
751 /** Variable index if holding a variable, otherwise UINT8_MAX. */
752 uint8_t idxVar;
753 /** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
754 * by iemNativeVarRestoreVolatileRegsPostHlpCall. This is not valid outside
755 * that scope. */
756 uint8_t idxStackSlot;
757 /** Alignment padding. */
758 uint8_t abAlign[5];
759} IEMNATIVEHSTREG;
760
761
762/**
763 * Core state for the native recompiler, that is, things that needs careful
764 * handling when dealing with branches.
765 */
766typedef struct IEMNATIVECORESTATE
767{
768 /** Allocation bitmap for aHstRegs. */
769 uint32_t bmHstRegs;
770
771 /** Bitmap marking which host register contains guest register shadow copies.
772 * This is used during register allocation to try preserve copies. */
773 uint32_t bmHstRegsWithGstShadow;
774 /** Bitmap marking valid entries in aidxGstRegShadows. */
775 uint64_t bmGstRegShadows;
776
777 union
778 {
779 /** Index of variable arguments, UINT8_MAX if not valid. */
780 uint8_t aidxArgVars[8];
781 /** For more efficient resetting. */
782 uint64_t u64ArgVars;
783 };
784
785 /** Allocation bitmap for the stack. */
786 uint32_t bmStack;
787 /** Allocation bitmap for aVars. */
788 uint32_t bmVars;
789
790 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
791 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
792 * (A shadow copy of a guest register can only be held in a one host register,
793 * there are no duplicate copies or ambiguities like that). */
794 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
795
796 /** Host register allocation tracking. */
797 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
798
799 /** Variables and arguments. */
800 IEMNATIVEVAR aVars[9];
801} IEMNATIVECORESTATE;
802/** Pointer to core state. */
803typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;
804/** Pointer to const core state. */
805typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
806
807
808/**
809 * Conditional stack entry.
810 */
811typedef struct IEMNATIVECOND
812{
813 /** Set if we're in the "else" part, clear if we're in the "if" before it. */
814 bool fInElse;
815 /** The label for the IEM_MC_ELSE. */
816 uint32_t idxLabelElse;
817 /** The label for the IEM_MC_ENDIF. */
818 uint32_t idxLabelEndIf;
819 /** The initial state snapshot as the if-block starts executing. */
820 IEMNATIVECORESTATE InitialState;
821 /** The state snapshot at the end of the if-block. */
822 IEMNATIVECORESTATE IfFinalState;
823} IEMNATIVECOND;
824/** Pointer to a condition stack entry. */
825typedef IEMNATIVECOND *PIEMNATIVECOND;
826
827
828/**
829 * Native recompiler state.
830 */
831typedef struct IEMRECOMPILERSTATE
832{
833 /** Size of the buffer that pbNativeRecompileBufR3 points to in
834 * IEMNATIVEINSTR units. */
835 uint32_t cInstrBufAlloc;
836#ifdef VBOX_STRICT
837 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */
838 uint32_t offInstrBufChecked;
839#else
840 uint32_t uPadding1; /* We don't keep track of the size here... */
841#endif
842 /** Fixed temporary code buffer for native recompilation. */
843 PIEMNATIVEINSTR pInstrBuf;
844
845 /** Bitmaps with the label types used. */
846 uint64_t bmLabelTypes;
847 /** Actual number of labels in paLabels. */
848 uint32_t cLabels;
849 /** Max number of entries allowed in paLabels before reallocating it. */
850 uint32_t cLabelsAlloc;
851 /** Labels defined while recompiling (referenced by fixups). */
852 PIEMNATIVELABEL paLabels;
853 /** Array with indexes of unique labels (uData always 0). */
854 uint32_t aidxUniqueLabels[kIemNativeLabelType_FirstWithMultipleInstances];
855
856 /** Actual number of fixups paFixups. */
857 uint32_t cFixups;
858 /** Max number of entries allowed in paFixups before reallocating it. */
859 uint32_t cFixupsAlloc;
860 /** Buffer used by the recompiler for recording fixups when generating code. */
861 PIEMNATIVEFIXUP paFixups;
862
863#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
864 /** Number of debug info entries allocated for pDbgInfo. */
865 uint32_t cDbgInfoAlloc;
866 uint32_t uPadding;
867 /** Debug info. */
868 PIEMTBDBG pDbgInfo;
869#endif
870
871#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
872 /** The current call index (liveness array and threaded calls in TB). */
873 uint32_t idxCurCall;
874 /** Number of liveness entries allocated. */
875 uint32_t cLivenessEntriesAlloc;
876 /** Liveness entries for all the calls in the TB begin recompiled.
877 * The entry for idxCurCall contains the info for what the next call will
878 * require wrt registers. (Which means the last entry is the initial liveness
879 * state.) */
880 PIEMLIVENESSENTRY paLivenessEntries;
881#endif
882
883 /** The translation block being recompiled. */
884 PCIEMTB pTbOrg;
885 /** The VMCPU structure of the EMT. */
886 PVMCPUCC pVCpu;
887
888 /** Condition sequence number (for generating unique labels). */
889 uint16_t uCondSeqNo;
890 /** Check IRQ seqeunce number (for generating unique labels). */
891 uint16_t uCheckIrqSeqNo;
892 /** TLB load sequence number (for generating unique labels). */
893 uint16_t uTlbSeqNo;
894 /** The current condition stack depth (aCondStack). */
895 uint8_t cCondDepth;
896
897 /** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
898 uint8_t cArgs;
899 /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
900 uint32_t fCImpl;
901 /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
902 uint32_t fMc;
903 /** The expected IEMCPU::fExec value for the current call/instruction. */
904 uint32_t fExec;
905
906 /** Core state requiring care with branches. */
907 IEMNATIVECORESTATE Core;
908
909 /** The condition nesting stack. */
910 IEMNATIVECOND aCondStack[2];
911
912#ifndef IEM_WITH_THROW_CATCH
913 /** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions
914 * for recompilation error handling. */
915 jmp_buf JmpBuf;
916#endif
917} IEMRECOMPILERSTATE;
918/** Pointer to a native recompiler state. */
919typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
920
921
922/** @def IEMNATIVE_TRY_SETJMP
923 * Wrapper around setjmp / try, hiding all the ugly differences.
924 *
925 * @note Use with extreme care as this is a fragile macro.
926 * @param a_pReNative The native recompile state.
927 * @param a_rcTarget The variable that should receive the status code in case
928 * of a longjmp/throw.
929 */
930/** @def IEMNATIVE_CATCH_LONGJMP_BEGIN
931 * Start wrapper for catch / setjmp-else.
932 *
933 * This will set up a scope.
934 *
935 * @note Use with extreme care as this is a fragile macro.
936 * @param a_pReNative The native recompile state.
937 * @param a_rcTarget The variable that should receive the status code in case
938 * of a longjmp/throw.
939 */
940/** @def IEMNATIVE_CATCH_LONGJMP_END
941 * End wrapper for catch / setjmp-else.
942 *
943 * This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean
944 * up the state.
945 *
946 * @note Use with extreme care as this is a fragile macro.
947 * @param a_pReNative The native recompile state.
948 */
949/** @def IEMNATIVE_DO_LONGJMP
950 *
951 * Wrapper around longjmp / throw.
952 *
953 * @param a_pReNative The native recompile state.
954 * @param a_rc The status code jump back with / throw.
955 */
956#ifdef IEM_WITH_THROW_CATCH
957# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
958 a_rcTarget = VINF_SUCCESS; \
959 try
960# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
961 catch (int rcThrown) \
962 { \
963 a_rcTarget = rcThrown
964# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
965 } \
966 ((void)0)
967# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)
968#else /* !IEM_WITH_THROW_CATCH */
969# define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \
970 if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)
971# define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \
972 else \
973 { \
974 ((void)0)
975# define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \
976 }
977# define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))
978#endif /* !IEM_WITH_THROW_CATCH */
979
980
981/**
982 * Native recompiler worker for a threaded function.
983 *
984 * @returns New code buffer offset; throws VBox status code in case of a failure.
985 * @param pReNative The native recompiler state.
986 * @param off The current code buffer offset.
987 * @param pCallEntry The threaded call entry.
988 *
989 * @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!
990 */
991typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);
992/** Pointer to a native recompiler worker for a threaded function. */
993typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
994
995/** Defines a native recompiler worker for a threaded function.
996 * @see FNIEMNATIVERECOMPFUNC */
997#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
998 uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
999
1000/** Prototypes a native recompiler function for a threaded function.
1001 * @see FNIEMNATIVERECOMPFUNC */
1002#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
1003
1004
1005/**
1006 * Native recompiler liveness analysis worker for a threaded function.
1007 *
1008 * @param pCallEntry The threaded call entry.
1009 * @param pIncoming The incoming liveness state entry.
1010 * @param pOutgoing The outgoing liveness state entry.
1011 */
1012typedef DECLCALLBACKTYPE(void, FNIEMNATIVELIVENESSFUNC, (PCIEMTHRDEDCALLENTRY pCallEntry,
1013 PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing));
1014/** Pointer to a native recompiler liveness analysis worker for a threaded function. */
1015typedef FNIEMNATIVELIVENESSFUNC *PFNIEMNATIVELIVENESSFUNC;
1016
1017/** Defines a native recompiler liveness analysis worker for a threaded function.
1018 * @see FNIEMNATIVELIVENESSFUNC */
1019#define IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(a_Name) \
1020 DECLCALLBACK(void) a_Name(PCIEMTHRDEDCALLENTRY pCallEntry, PCIEMLIVENESSENTRY pIncoming, PIEMLIVENESSENTRY pOutgoing)
1021
1022/** Prototypes a native recompiler liveness analysis function for a threaded function.
1023 * @see FNIEMNATIVELIVENESSFUNC */
1024#define IEM_DECL_IEMNATIVELIVENESSFUNC_PROTO(a_Name) FNIEMNATIVELIVENESSFUNC a_Name
1025
1026
1027/** Define a native recompiler helper function, safe to call from the TB code. */
1028#define IEM_DECL_NATIVE_HLP_DEF(a_RetType, a_Name, a_ArgList) \
1029 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1030/** Prototype a native recompiler helper function, safe to call from the TB code. */
1031#define IEM_DECL_NATIVE_HLP_PROTO(a_RetType, a_Name, a_ArgList) \
1032 DECL_HIDDEN_THROW(a_RetType) VBOXCALL a_Name a_ArgList
1033
1034
1035DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
1036 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
1037DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
1038DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
1039 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
1040DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
1041
1042DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);
1043DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpEx(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint32_t fRegMask,
1044 bool fPreferVolatile = true);
1045DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
1046 bool fPreferVolatile = true);
1047DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1048 IEMNATIVEGSTREG enmGstReg,
1049 IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
1050 bool fNoVolatileRegs = false);
1051DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
1052 IEMNATIVEGSTREG enmGstReg);
1053
1054DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar);
1055DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
1056DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
1057DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1058DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1059DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
1060DECLHIDDEN(void) iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
1061DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
1062DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off);
1063DECL_HIDDEN_THROW(uint32_t) iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs,
1064 uint32_t fKeepVars = 0);
1065DECLHIDDEN(void) iemNativeRegFlushGuestShadows(PIEMRECOMPILERSTATE pReNative, uint64_t fGstRegs) RT_NOEXCEPT;
1066DECLHIDDEN(void) iemNativeRegFlushGuestShadowsByHostMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegs) RT_NOEXCEPT;
1067DECL_HIDDEN_THROW(uint32_t) iemNativeRegRestoreGuestShadowsInVolatileRegs(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1068 uint32_t fHstRegsActiveShadows);
1069
1070DECL_HIDDEN_THROW(uint8_t) iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
1071DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
1072 bool fInitialized = false, uint8_t idxRegPref = UINT8_MAX);
1073DECL_HIDDEN_THROW(uint8_t) iemNativeVarRegisterAcquireForGuestReg(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar,
1074 IEMNATIVEGSTREG enmGstReg, uint32_t *poff);
1075DECL_HIDDEN_THROW(uint32_t) iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1076 uint32_t fHstRegsNotToSave);
1077DECL_HIDDEN_THROW(uint32_t) iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1078 uint32_t fHstRegsNotToSave);
1079
1080DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1081 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);
1082DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);
1083DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCImplCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
1084 uint64_t fGstShwFlush, uintptr_t pfnCImpl, uint8_t cbInstr, uint8_t cAddParams,
1085 uint64_t uParam0, uint64_t uParam1, uint64_t uParam2);
1086DECL_HIDDEN_THROW(uint32_t) iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off,
1087 PCIEMTHRDEDCALLENTRY pCallEntry);
1088
1089extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];
1090
1091
1092/**
1093 * Ensures that there is sufficient space in the instruction output buffer.
1094 *
1095 * This will reallocate the buffer if needed and allowed.
1096 *
1097 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the
1098 * allocation size.
1099 *
1100 * @returns Pointer to the instruction output buffer on success; throws VBox
1101 * status code on failure, so no need to check it.
1102 * @param pReNative The native recompile state.
1103 * @param off Current instruction offset. Works safely for UINT32_MAX
1104 * as well.
1105 * @param cInstrReq Number of instruction about to be added. It's okay to
1106 * overestimate this a bit.
1107 */
1108DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)
1109iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
1110{
1111 uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */
1112 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))
1113 {
1114#ifdef VBOX_STRICT
1115 pReNative->offInstrBufChecked = offChecked;
1116#endif
1117 return pReNative->pInstrBuf;
1118 }
1119 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
1120}
1121
1122/**
1123 * Checks that we didn't exceed the space requested in the last
1124 * iemNativeInstrBufEnsure() call.
1125 */
1126#define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \
1127 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \
1128 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))
1129
1130/**
1131 * Checks that a variable index is valid.
1132 */
1133#define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
1134 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1135 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
1136
1137/**
1138 * Checks that a variable index is valid and that the variable is assigned the
1139 * correct argument number.
1140 * This also adds a RT_NOREF of a_idxVar.
1141 */
1142#define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
1143 RT_NOREF_PV(a_idxVar); \
1144 AssertMsg( (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
1145 && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar))\
1146 && (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
1147 , ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
1148 (a_pReNative)->Core.aVars[RT_MAX(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
1149 } while (0)
1150
1151/**
1152 * Calculates the stack address of a variable as a [r]BP displacement value.
1153 */
1154DECL_FORCE_INLINE(int32_t)
1155iemNativeStackCalcBpDisp(uint8_t idxStackSlot)
1156{
1157 Assert(idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS);
1158 return idxStackSlot * sizeof(uint64_t) + IEMNATIVE_FP_OFF_STACK_VARS;
1159}
1160
1161
1162/**
1163 * Releases the variable's register.
1164 *
1165 * The register must have been previously acquired calling
1166 * iemNativeVarRegisterAcquire(), iemNativeVarRegisterAcquireForGuestReg() or
1167 * iemNativeVarRegisterSetAndAcquire().
1168 */
1169DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
1170{
1171 IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
1172 Assert(pReNative->Core.aVars[idxVar].fRegAcquired);
1173 pReNative->Core.aVars[idxVar].fRegAcquired = false;
1174}
1175
1176/** @} */
1177
1178#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
1179
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette