VirtualBox

source: vbox/trunk/include/VBox/dis-x86-amd64.h

Last change on this file was 103512, checked in by vboxsync, 3 months ago

DIS: Added vex group 13 and redid some or the vex prefix decoding.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 18.7 KB
Line 
1/** @file
2 * DIS - The VirtualBox Disassembler.
3 */
4
5/*
6 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
7 *
8 * This file is part of VirtualBox base platform packages, as
9 * available from https://www.virtualbox.org.
10 *
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation, in version 3 of the
14 * License.
15 *
16 * This program is distributed in the hope that it will be useful, but
17 * WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, see <https://www.gnu.org/licenses>.
23 *
24 * The contents of this file may alternatively be used under the terms
25 * of the Common Development and Distribution License Version 1.0
26 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
27 * in the VirtualBox distribution, in which case the provisions of the
28 * CDDL are applicable instead of those of the GPL.
29 *
30 * You may elect to license modified versions of this file under the
31 * terms and conditions of either the GPL or the CDDL or both.
32 *
33 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
34 */
35
36#ifndef VBOX_INCLUDED_dis_x86_amd64_h
37#define VBOX_INCLUDED_dis_x86_amd64_h
38#ifndef RT_WITHOUT_PRAGMA_ONCE
39# pragma once
40#endif
41
42#include <VBox/types.h>
43#include <VBox/disopcode-x86-amd64.h>
44#include <iprt/assert.h>
45
46
47RT_C_DECLS_BEGIN
48
49/** @addtogroup grp_dis VBox Disassembler
50 * @{ */
51
52/** @name Prefix byte flags (DISSTATE::fPrefix).
53 * @{
54 */
55#define DISPREFIX_NONE UINT8_C(0x00)
56/** non-default address size. */
57#define DISPREFIX_ADDRSIZE UINT8_C(0x01)
58/** non-default operand size. */
59#define DISPREFIX_OPSIZE UINT8_C(0x02)
60/** lock prefix. */
61#define DISPREFIX_LOCK UINT8_C(0x04)
62/** segment prefix. */
63#define DISPREFIX_SEG UINT8_C(0x08)
64/** rep(e) prefix (not a prefix, but we'll treat is as one). */
65#define DISPREFIX_REP UINT8_C(0x10)
66/** repne prefix (not a prefix, but we'll treat is as one). */
67#define DISPREFIX_REPNE UINT8_C(0x20)
68/** REX prefix (64 bits) */
69#define DISPREFIX_REX UINT8_C(0x40)
70/** VEX or EVEX prefix (64 bits) */
71#define DISPREFIX_VEX UINT8_C(0x80)
72/** @} */
73
74/** @name VEX.Lvvvv prefix destination register flag.
75 * @todo r=bird: redo this. See comments elsewhere.
76 * @{
77 */
78#define VEX_LEN256 UINT8_C(0x01)
79#define VEXREG_IS256B(x) ((x) & VEX_LEN256)
80/* Convert second byte of VEX prefix to internal format */
81#define VEX_2B2INT(x) ((((x) >> 2) & 0x1f))
82#define VEX_HAS_REX_R(x) (!((x) & 0x80))
83#define DISPREFIX_VEX_FLAG_W UINT8_C(0x01)
84/** @} */
85
86/** @name DISPREFIX_VEX_F_XXX - The bVexStuff definitions
87 * @note This is the same layout as byte2 in the VEX3 prefix.
88 * @{ */
89/** The VEX prefix mask. */
90#define DISPREFIX_VEX_F_PP_MASK UINT8_C(0x03)
91/** No prefix. */
92#define DISPREFIX_VEX_F_PP_NONE UINT8_C(0x00)
93/** 66h/opsize prefix. */
94#define DISPREFIX_VEX_F_PP_66 UINT8_C(0x01)
95/** f3h/repe prefix. */
96#define DISPREFIX_VEX_F_PP_F3 UINT8_C(0x02)
97/** f2h/repne prefix. */
98#define DISPREFIX_VEX_F_PP_F2 UINT8_C(0x03)
99/** The VEX.L flag. */
100#define DISPREFIX_VEX_F_L UINT8_C(0x04)
101/** The VEX.VVVV mask. */
102#define DISPREFIX_VEX_F_VVVV UINT8_C(0x78)
103/** The VEX.VVVV shift count. */
104#define DISPREFIX_VEX_F_VVVV_SHIFT 3
105/** The VEX.VVVV mask shifted down to bit 0. */
106#define DISPREFIX_VEX_F_VVVV_SMASK UINT8_C(0x0f)
107/** The VEX.W flag. */
108#define DISPREFIX_VEX_F_W UINT8_C(0x80)
109 /** @} */
110
111/** @name 64 bits prefix byte flags (DISSTATE::fRexPrefix).
112 * Requires VBox/disopcode.h.
113 * @{
114 */
115#define DISPREFIX_REX_OP_2_FLAGS(a) (a - OP_PARM_REX_START)
116/*#define DISPREFIX_REX_FLAGS DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX) - 0, which is no flag */
117#define DISPREFIX_REX_FLAGS_B DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_B)
118#define DISPREFIX_REX_FLAGS_X DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_X)
119#define DISPREFIX_REX_FLAGS_XB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_XB)
120#define DISPREFIX_REX_FLAGS_R DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_R)
121#define DISPREFIX_REX_FLAGS_RB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_RB)
122#define DISPREFIX_REX_FLAGS_RX DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_RX)
123#define DISPREFIX_REX_FLAGS_RXB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_RXB)
124#define DISPREFIX_REX_FLAGS_W DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_W)
125#define DISPREFIX_REX_FLAGS_WB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WB)
126#define DISPREFIX_REX_FLAGS_WX DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WX)
127#define DISPREFIX_REX_FLAGS_WXB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WXB)
128#define DISPREFIX_REX_FLAGS_WR DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WR)
129#define DISPREFIX_REX_FLAGS_WRB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WRB)
130#define DISPREFIX_REX_FLAGS_WRX DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WRX)
131#define DISPREFIX_REX_FLAGS_WRXB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WRXB)
132/** @} */
133AssertCompile(RT_IS_POWER_OF_TWO(DISPREFIX_REX_FLAGS_B));
134AssertCompile(RT_IS_POWER_OF_TWO(DISPREFIX_REX_FLAGS_X));
135AssertCompile(RT_IS_POWER_OF_TWO(DISPREFIX_REX_FLAGS_W));
136AssertCompile(RT_IS_POWER_OF_TWO(DISPREFIX_REX_FLAGS_R));
137
138
139/** @name 64-bit general register indexes.
140 * This matches the AMD64 register encoding. It is found used in
141 * DISOPPARAM::Base.idxGenReg and DISOPPARAM::Index.idxGenReg.
142 * @note Safe to assume same values as the 16-bit and 32-bit general registers.
143 * @{
144 */
145#define DISGREG_RAX UINT8_C(0)
146#define DISGREG_RCX UINT8_C(1)
147#define DISGREG_RDX UINT8_C(2)
148#define DISGREG_RBX UINT8_C(3)
149#define DISGREG_RSP UINT8_C(4)
150#define DISGREG_RBP UINT8_C(5)
151#define DISGREG_RSI UINT8_C(6)
152#define DISGREG_RDI UINT8_C(7)
153#define DISGREG_R8 UINT8_C(8)
154#define DISGREG_R9 UINT8_C(9)
155#define DISGREG_R10 UINT8_C(10)
156#define DISGREG_R11 UINT8_C(11)
157#define DISGREG_R12 UINT8_C(12)
158#define DISGREG_R13 UINT8_C(13)
159#define DISGREG_R14 UINT8_C(14)
160#define DISGREG_R15 UINT8_C(15)
161/** @} */
162
163/** @name 32-bit general register indexes.
164 * This matches the AMD64 register encoding. It is found used in
165 * DISOPPARAM::Base.idxGenReg and DISOPPARAM::Index.idxGenReg.
166 * @note Safe to assume same values as the 16-bit and 64-bit general registers.
167 * @{
168 */
169#define DISGREG_EAX UINT8_C(0)
170#define DISGREG_ECX UINT8_C(1)
171#define DISGREG_EDX UINT8_C(2)
172#define DISGREG_EBX UINT8_C(3)
173#define DISGREG_ESP UINT8_C(4)
174#define DISGREG_EBP UINT8_C(5)
175#define DISGREG_ESI UINT8_C(6)
176#define DISGREG_EDI UINT8_C(7)
177#define DISGREG_R8D UINT8_C(8)
178#define DISGREG_R9D UINT8_C(9)
179#define DISGREG_R10D UINT8_C(10)
180#define DISGREG_R11D UINT8_C(11)
181#define DISGREG_R12D UINT8_C(12)
182#define DISGREG_R13D UINT8_C(13)
183#define DISGREG_R14D UINT8_C(14)
184#define DISGREG_R15D UINT8_C(15)
185/** @} */
186
187/** @name 16-bit general register indexes.
188 * This matches the AMD64 register encoding. It is found used in
189 * DISOPPARAM::Base.idxGenReg and DISOPPARAM::Index.idxGenReg.
190 * @note Safe to assume same values as the 32-bit and 64-bit general registers.
191 * @{
192 */
193#define DISGREG_AX UINT8_C(0)
194#define DISGREG_CX UINT8_C(1)
195#define DISGREG_DX UINT8_C(2)
196#define DISGREG_BX UINT8_C(3)
197#define DISGREG_SP UINT8_C(4)
198#define DISGREG_BP UINT8_C(5)
199#define DISGREG_SI UINT8_C(6)
200#define DISGREG_DI UINT8_C(7)
201#define DISGREG_R8W UINT8_C(8)
202#define DISGREG_R9W UINT8_C(9)
203#define DISGREG_R10W UINT8_C(10)
204#define DISGREG_R11W UINT8_C(11)
205#define DISGREG_R12W UINT8_C(12)
206#define DISGREG_R13W UINT8_C(13)
207#define DISGREG_R14W UINT8_C(14)
208#define DISGREG_R15W UINT8_C(15)
209/** @} */
210
211/** @name 8-bit general register indexes.
212 * This mostly (?) matches the AMD64 register encoding. It is found used in
213 * DISOPPARAM::Base.idxGenReg and DISOPPARAM::Index.idxGenReg.
214 * @{
215 */
216#define DISGREG_AL UINT8_C(0)
217#define DISGREG_CL UINT8_C(1)
218#define DISGREG_DL UINT8_C(2)
219#define DISGREG_BL UINT8_C(3)
220#define DISGREG_AH UINT8_C(4)
221#define DISGREG_CH UINT8_C(5)
222#define DISGREG_DH UINT8_C(6)
223#define DISGREG_BH UINT8_C(7)
224#define DISGREG_R8B UINT8_C(8)
225#define DISGREG_R9B UINT8_C(9)
226#define DISGREG_R10B UINT8_C(10)
227#define DISGREG_R11B UINT8_C(11)
228#define DISGREG_R12B UINT8_C(12)
229#define DISGREG_R13B UINT8_C(13)
230#define DISGREG_R14B UINT8_C(14)
231#define DISGREG_R15B UINT8_C(15)
232#define DISGREG_SPL UINT8_C(16)
233#define DISGREG_BPL UINT8_C(17)
234#define DISGREG_SIL UINT8_C(18)
235#define DISGREG_DIL UINT8_C(19)
236/** @} */
237
238/** @name Segment registerindexes.
239 * This matches the AMD64 register encoding. It is found used in
240 * DISOPPARAM::Base.idxSegReg.
241 * @{
242 */
243typedef enum
244{
245 DISSELREG_ES = 0,
246 DISSELREG_CS = 1,
247 DISSELREG_SS = 2,
248 DISSELREG_DS = 3,
249 DISSELREG_FS = 4,
250 DISSELREG_GS = 5,
251 /** End of the valid register index values. */
252 DISSELREG_END,
253 /** The usual 32-bit paranoia. */
254 DIS_SEGREG_32BIT_HACK = 0x7fffffff
255} DISSELREG;
256/** @} */
257
258/** @name FPU register indexes.
259 * This matches the AMD64 register encoding. It is found used in
260 * DISOPPARAM::Base.idxFpuReg.
261 * @{
262 */
263#define DISFPREG_ST0 UINT8_C(0)
264#define DISFPREG_ST1 UINT8_C(1)
265#define DISFPREG_ST2 UINT8_C(2)
266#define DISFPREG_ST3 UINT8_C(3)
267#define DISFPREG_ST4 UINT8_C(4)
268#define DISFPREG_ST5 UINT8_C(5)
269#define DISFPREG_ST6 UINT8_C(6)
270#define DISFPREG_ST7 UINT8_C(7)
271/** @} */
272
273/** @name Control register indexes.
274 * This matches the AMD64 register encoding. It is found used in
275 * DISOPPARAM::Base.idxCtrlReg.
276 * @{
277 */
278#define DISCREG_CR0 UINT8_C(0)
279#define DISCREG_CR1 UINT8_C(1)
280#define DISCREG_CR2 UINT8_C(2)
281#define DISCREG_CR3 UINT8_C(3)
282#define DISCREG_CR4 UINT8_C(4)
283#define DISCREG_CR8 UINT8_C(8)
284/** @} */
285
286/** @name Debug register indexes.
287 * This matches the AMD64 register encoding. It is found used in
288 * DISOPPARAM::Base.idxDbgReg.
289 * @{
290 */
291#define DISDREG_DR0 UINT8_C(0)
292#define DISDREG_DR1 UINT8_C(1)
293#define DISDREG_DR2 UINT8_C(2)
294#define DISDREG_DR3 UINT8_C(3)
295#define DISDREG_DR4 UINT8_C(4)
296#define DISDREG_DR5 UINT8_C(5)
297#define DISDREG_DR6 UINT8_C(6)
298#define DISDREG_DR7 UINT8_C(7)
299/** @} */
300
301/** @name MMX register indexes.
302 * This matches the AMD64 register encoding. It is found used in
303 * DISOPPARAM::Base.idxMmxReg.
304 * @{
305 */
306#define DISMREG_MMX0 UINT8_C(0)
307#define DISMREG_MMX1 UINT8_C(1)
308#define DISMREG_MMX2 UINT8_C(2)
309#define DISMREG_MMX3 UINT8_C(3)
310#define DISMREG_MMX4 UINT8_C(4)
311#define DISMREG_MMX5 UINT8_C(5)
312#define DISMREG_MMX6 UINT8_C(6)
313#define DISMREG_MMX7 UINT8_C(7)
314/** @} */
315
316/** @name SSE register indexes.
317 * This matches the AMD64 register encoding. It is found used in
318 * DISOPPARAM::Base.idxXmmReg.
319 * @{
320 */
321#define DISXREG_XMM0 UINT8_C(0)
322#define DISXREG_XMM1 UINT8_C(1)
323#define DISXREG_XMM2 UINT8_C(2)
324#define DISXREG_XMM3 UINT8_C(3)
325#define DISXREG_XMM4 UINT8_C(4)
326#define DISXREG_XMM5 UINT8_C(5)
327#define DISXREG_XMM6 UINT8_C(6)
328#define DISXREG_XMM7 UINT8_C(7)
329/** @} */
330
331
332/**
333 * Opcode parameter (operand) details for x86/AMD64.
334 */
335typedef struct
336{
337 /** Disposition. */
338 union
339 {
340 /** 64-bit displacement, applicable if DISUSE_DISPLACEMENT64 is set in fUse. */
341 int64_t i64;
342 uint64_t u64;
343 /** 32-bit displacement, applicable if DISUSE_DISPLACEMENT32 or
344 * DISUSE_RIPDISPLACEMENT32 is set in fUse. */
345 int32_t i32;
346 uint32_t u32;
347 /** 16-bit displacement, applicable if DISUSE_DISPLACEMENT16 is set in fUse. */
348 int32_t i16;
349 uint32_t u16;
350 /** 8-bit displacement, applicable if DISUSE_DISPLACEMENT8 is set in fUse. */
351 int32_t i8;
352 uint32_t u8;
353 } uDisp;
354 /** The base register from ModR/M or SIB, applicable if DISUSE_BASE is
355 * set in fUse. */
356 union
357 {
358 /** General register index (DISGREG_XXX), applicable if DISUSE_REG_GEN8,
359 * DISUSE_REG_GEN16, DISUSE_REG_GEN32 or DISUSE_REG_GEN64 is set in fUse. */
360 uint8_t idxGenReg;
361 /** FPU stack register index (DISFPREG_XXX), applicable if DISUSE_REG_FP is
362 * set in fUse. 1:1 indexes. */
363 uint8_t idxFpuReg;
364 /** MMX register index (DISMREG_XXX), applicable if DISUSE_REG_MMX is
365 * set in fUse. 1:1 indexes. */
366 uint8_t idxMmxReg;
367 /** SSE register index (DISXREG_XXX), applicable if DISUSE_REG_XMM is
368 * set in fUse. 1:1 indexes. */
369 uint8_t idxXmmReg;
370 /** SSE2 register index (DISYREG_XXX), applicable if DISUSE_REG_YMM is
371 * set in fUse. 1:1 indexes. */
372 uint8_t idxYmmReg;
373 /** Segment register index (DISSELREG_XXX), applicable if DISUSE_REG_SEG is
374 * set in fUse. */
375 uint8_t idxSegReg;
376 /** Test register, TR0-TR7, present on early IA32 CPUs, applicable if
377 * DISUSE_REG_TEST is set in fUse. No index defines for these. */
378 uint8_t idxTestReg;
379 /** Control register index (DISCREG_XXX), applicable if DISUSE_REG_CR is
380 * set in fUse. 1:1 indexes. */
381 uint8_t idxCtrlReg;
382 /** Debug register index (DISDREG_XXX), applicable if DISUSE_REG_DBG is
383 * set in fUse. 1:1 indexes. */
384 uint8_t idxDbgReg;
385 } Base;
386 /** The SIB index register meaning, applicable if DISUSE_INDEX is
387 * set in fUse. */
388 union
389 {
390 /** General register index (DISGREG_XXX), applicable if DISUSE_REG_GEN8,
391 * DISUSE_REG_GEN16, DISUSE_REG_GEN32 or DISUSE_REG_GEN64 is set in fUse. */
392 uint8_t idxGenReg;
393 /** XMM register index (DISXREG_XXX), applicable if DISUSE_REG_XMM
394 * is set in fUse. */
395 uint8_t idxXmmReg;
396 /** YMM register index (DISXREG_XXX), applicable if DISUSE_REG_YMM
397 * is set in fUse. */
398 uint8_t idxYmmReg;
399 } Index;
400 /** 2, 4 or 8, if DISUSE_SCALE is set in fUse. */
401 uint8_t uScale;
402 /** Parameter size. */
403 uint8_t cb;
404 /** Copy of the corresponding DISOPCODE::fParam1 / DISOPCODE::fParam2 /
405 * DISOPCODE::fParam3. */
406 uint32_t fParam;
407} DIS_OP_PARAM_X86_T;
408AssertCompileSize(DIS_OP_PARAM_X86_T, 16);
409/** Pointer to opcode parameter. */
410typedef DIS_OP_PARAM_X86_T *PDIS_OP_PARAM_X86_T;
411/** Pointer to opcode parameter. */
412typedef const DIS_OP_PARAM_X86_T *PCDIS_OP_PARAM_X86_T;
413
414
415/** Parser callback.
416 * @remark no DECLCALLBACK() here because it's considered to be internal and
417 * there is no point in enforcing CDECL. */
418typedef size_t FNDISPARSEX86(size_t offInstr, PCDISOPCODE pOp, PDISSTATE pDis, PDISOPPARAM pParam);
419/** Pointer to a disassembler parser function. */
420typedef FNDISPARSEX86 *PFNDISPARSEX86;
421/** Pointer to a const disassembler parser function pointer. */
422typedef PFNDISPARSEX86 const *PCPFNDISPARSEX86;
423
424/**
425 * The x86/AMD64 specific disassembler state and result.
426 */
427typedef struct
428{
429 /** SIB fields. */
430 union
431 {
432 /** Bitfield view */
433 struct
434 {
435 uint8_t Base;
436 uint8_t Index;
437 uint8_t Scale;
438 } Bits;
439 } SIB;
440 /** ModRM fields. */
441 union
442 {
443 /** Bitfield view */
444 struct
445 {
446 uint8_t Rm;
447 uint8_t Reg;
448 uint8_t Mod;
449 } Bits;
450 } ModRM;
451 /** The addressing mode (DISCPUMODE). */
452 uint8_t uAddrMode;
453 /** The operand mode (DISCPUMODE). */
454 uint8_t uOpMode;
455 /** Per instruction prefix settings. */
456 uint8_t fPrefix;
457 /** REX prefix value (64 bits only). */
458 uint8_t fRexPrefix;
459 /** Segment prefix value (DISSELREG). */
460 uint8_t idxSegPrefix;
461 /** Last prefix byte (for SSE2 extension tables). */
462 uint8_t bLastPrefix;
463 /** Last significant opcode byte of instruction. */
464 uint8_t bOpCode;
465 /** The size of the prefix bytes. */
466 uint8_t cbPrefix;
467 /** VEX presence flag, destination register and size
468 * @todo r=bird: There is no VEX presence flage here, just ~vvvv and L.
469 * @deprecated DO NOT USE. All info is available in bVexByte2 and
470 * fPrefix/DISPREFIX_VEX. */
471 uint8_t bVexDestReg;
472 /** DISPREFIX_VEX_F_XXX - This is set for both VEX prefixes. */
473 uint8_t bVexByte2;
474 /** Internal: instruction filter */
475 uint32_t fFilter;
476 /** SIB displacment. */
477 int32_t i32SibDisp;
478 /** Internal: pointer to disassembly function table */
479 PCPFNDISPARSEX86 pfnDisasmFnTable;
480#if ARCH_BITS == 32
481 uint32_t uPtrPadding1;
482#endif
483} DIS_STATE_X86_T;
484AssertCompileSize(DIS_STATE_X86_T, 32);
485
486
487
488DISDECL(bool) DISFormatYasmIsOddEncoding(PDISSTATE pDis);
489
490/** @} */
491
492RT_C_DECLS_END
493
494#endif /* !VBOX_INCLUDED_dis_x86_amd64_h */
495
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use