IEMInline.h@ 102585

Last change on this file since 102585 was 102585, checked in by vboxsync, 17 months ago
VMM/IEM: Refactored the IEM_MC_SET_RIP_Uxx_AND_FINISH MCs in prep for native translation. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 140.5 KB

Line
1	/* $Id: IEMInline.h 102585 2023-12-12 12:26:29Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	/* Deal with the cases that should be treated as VINF_SUCCESS first. */
49	if ( rcStrict == VINF_IEM_YIELD_PENDING_FF
50	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX /** @todo r=bird: Why do we need TWO status codes here? */
51	\|\| rcStrict == VINF_VMX_VMEXIT
52	#endif
53	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
54	\|\| rcStrict == VINF_SVM_VMEXIT
55	#endif
56	)
57	{
58	if (pVCpu->iem.s.rcPassUp == VINF_SUCCESS)
59	rcStrict = VINF_SUCCESS;
60	else
61	{
62	pVCpu->iem.s.cRetPassUpStatus++;
63	rcStrict = pVCpu->iem.s.rcPassUp;
64	}
65	}
66	else if (RT_SUCCESS(rcStrict))
67	{
68	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
69	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
70	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
71	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
72	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
73	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
74	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
75	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
76	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
77	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
78	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
79	\|\| rcStrict == VINF_EM_RAW_TO_R3
80	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
81	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
82	/* raw-mode / virt handlers only: */
83	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
84	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
85	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
86	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
87	\|\| rcStrict == VINF_SELM_SYNC_GDT
88	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
89	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
90	/* nested hw.virt codes: */
91	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
92	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
93	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
94	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
95	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
96	if (rcPassUp == VINF_SUCCESS)
97	pVCpu->iem.s.cRetInfStatuses++;
98	else if ( rcPassUp < VINF_EM_FIRST
99	\|\| rcPassUp > VINF_EM_LAST
100	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
101	{
102	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
103	pVCpu->iem.s.cRetPassUpStatus++;
104	rcStrict = rcPassUp;
105	}
106	else
107	{
108	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
109	pVCpu->iem.s.cRetInfStatuses++;
110	}
111	}
112	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
113	pVCpu->iem.s.cRetAspectNotImplemented++;
114	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
115	pVCpu->iem.s.cRetInstrNotImplemented++;
116	else
117	pVCpu->iem.s.cRetErrStatuses++;
118	}
119	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
120	{
121	pVCpu->iem.s.cRetPassUpStatus++;
122	rcStrict = pVCpu->iem.s.rcPassUp;
123	}
124
125	/* Just clear it here as well. */
126	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
127
128	return rcStrict;
129	}
130
131
132	/**
133	* Sets the pass up status.
134	*
135	* @returns VINF_SUCCESS.
136	* @param pVCpu The cross context virtual CPU structure of the
137	* calling thread.
138	* @param rcPassUp The pass up status. Must be informational.
139	* VINF_SUCCESS is not allowed.
140	*/
141	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
142	{
143	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
144
145	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
146	if (rcOldPassUp == VINF_SUCCESS)
147	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
148	/* If both are EM scheduling codes, use EM priority rules. */
149	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
150	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
151	{
152	if (rcPassUp < rcOldPassUp)
153	{
154	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
155	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
156	}
157	else
158	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
159	}
160	/* Override EM scheduling with specific status code. */
161	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
162	{
163	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
164	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
165	}
166	/* Don't override specific status code, first come first served. */
167	else
168	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
169	return VINF_SUCCESS;
170	}
171
172
173	/**
174	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.
175	*
176	* Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will
177	* reject expand down data segments and conforming code segments.
178	*
179	* ASSUMES that the CPU is in 32-bit mode.
180	*
181	* @note Will return zero when if any of the segment register state is marked
182	* external, this must be factored into assertions checking fExec
183	* consistency.
184	*
185	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
186	* @param pVCpu The cross context virtual CPU structure of the
187	* calling thread.
188	* @sa iemCalc32BitFlatIndicatorEsDs
189	*/
190	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT
191	{
192	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
193	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
194	\| pVCpu->cpum.GstCtx.cs.Attr.u
195	\| pVCpu->cpum.GstCtx.ss.Attr.u
196	\| pVCpu->cpum.GstCtx.ds.Attr.u)
197	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
198	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
199	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
200	\| (pVCpu->cpum.GstCtx.cs.u32Limit + 1)
201	\| (pVCpu->cpum.GstCtx.ss.u32Limit + 1)
202	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
203	== 0
204	&& ( pVCpu->cpum.GstCtx.es.u64Base
205	\| pVCpu->cpum.GstCtx.cs.u64Base
206	\| pVCpu->cpum.GstCtx.ss.u64Base
207	\| pVCpu->cpum.GstCtx.ds.u64Base)
208	== 0
209	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
210	? IEM_F_MODE_X86_32BIT_FLAT : 0;
211	}
212
213
214	/**
215	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are
216	* flat already.
217	*
218	* This is used by sysenter.
219	*
220	* @note Will return zero when if any of the segment register state is marked
221	* external, this must be factored into assertions checking fExec
222	* consistency.
223	*
224	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
225	* @param pVCpu The cross context virtual CPU structure of the
226	* calling thread.
227	* @sa iemCalc32BitFlatIndicator
228	*/
229	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT
230	{
231	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
232	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
233	\| pVCpu->cpum.GstCtx.ds.Attr.u)
234	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
235	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
236	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
237	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
238	== 0
239	&& ( pVCpu->cpum.GstCtx.es.u64Base
240	\| pVCpu->cpum.GstCtx.ds.u64Base)
241	== 0
242	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
243	? IEM_F_MODE_X86_32BIT_FLAT : 0;
244	}
245
246
247	/**
248	* Calculates the IEM_F_MODE_XXX and CPL flags.
249	*
250	* @returns IEM_F_MODE_XXX
251	* @param pVCpu The cross context virtual CPU structure of the
252	* calling thread.
253	*/
254	DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
255	{
256	/*
257	* We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx
258	* here to try get this done as efficiently as possible.
259	*/
260	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
261
262	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
263	{
264	if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
265	{
266	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
267	uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);
268	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
269	{
270	Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));
271	fExec \|= IEM_F_MODE_X86_32BIT_PROT \| iemCalc32BitFlatIndicator(pVCpu);
272	}
273	else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long
274	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))
275	fExec \|= IEM_F_MODE_X86_64BIT;
276	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
277	fExec \|= IEM_F_MODE_X86_16BIT_PROT;
278	else
279	fExec \|= IEM_F_MODE_X86_16BIT_PROT_PRE_386;
280	return fExec;
281	}
282	return IEM_F_MODE_X86_16BIT_PROT_V86 \| (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);
283	}
284
285	/* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
286	if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
287	{
288	if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
289	return IEM_F_MODE_X86_16BIT;
290	return IEM_F_MODE_X86_16BIT_PRE_386;
291	}
292
293	/* 32-bit unreal mode. */
294	return IEM_F_MODE_X86_32BIT \| iemCalc32BitFlatIndicator(pVCpu);
295	}
296
297
298	/**
299	* Calculates the AMD-V and VT-x related context flags.
300	*
301	* @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and
302	* IEM_F_X86_CTX_VMX.
303	* @param pVCpu The cross context virtual CPU structure of the
304	* calling thread.
305	*/
306	DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
307	{
308	/*
309	* This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled
310	* and CPUMIsGuestInNestedHwvirtMode to some extent.
311	*/
312	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
313
314	AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);
315	uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)
316	\| (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);
317	if (RT_LIKELY(!fTmp))
318	return 0; /* likely */
319
320	if (fTmp & X86_CR4_VMXE)
321	{
322	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
323	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)
324	return IEM_F_X86_CTX_VMX \| IEM_F_X86_CTX_IN_GUEST;
325	return IEM_F_X86_CTX_VMX;
326	}
327
328	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
329	if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)
330	return IEM_F_X86_CTX_SVM \| IEM_F_X86_CTX_IN_GUEST;
331	return IEM_F_X86_CTX_SVM;
332	}
333
334	#ifdef VBOX_INCLUDED_vmm_dbgf_h /* VM::dbgf.ro.cEnabledHwBreakpoints is only accessible if VBox/vmm/dbgf.h is included. */
335
336	/**
337	* Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.
338	*
339	* @returns IEM_F_BRK_PENDING_XXX or zero.
340	* @param pVCpu The cross context virtual CPU structure of the
341	* calling thread.
342	*/
343	DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
344	{
345	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
346
347	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
348	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
349	return 0;
350	return iemCalcExecDbgFlagsSlow(pVCpu);
351	}
352
353	/**
354	* Calculates the the IEM_F_XXX flags.
355	*
356	* @returns IEM_F_XXX combination match the current CPU state.
357	* @param pVCpu The cross context virtual CPU structure of the
358	* calling thread.
359	*/
360	DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
361	{
362	return iemCalcExecModeAndCplFlags(pVCpu)
363	\| iemCalcExecHwVirtFlags(pVCpu)
364	/* SMM is not yet implemented */
365	\| iemCalcExecDbgFlags(pVCpu)
366	;
367	}
368
369
370	/**
371	* Re-calculates the MODE and CPL parts of IEMCPU::fExec.
372	*
373	* @param pVCpu The cross context virtual CPU structure of the
374	* calling thread.
375	*/
376	DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplFlags(PVMCPUCC pVCpu)
377	{
378	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK))
379	\| iemCalcExecModeAndCplFlags(pVCpu);
380	}
381
382
383	/**
384	* Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.
385	*
386	* @param pVCpu The cross context virtual CPU structure of the
387	* calling thread.
388	*/
389	DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
390	{
391	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)
392	\| iemCalcExecDbgFlags(pVCpu);
393	}
394
395	#endif /* VBOX_INCLUDED_vmm_dbgf_h */
396
397
398	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
399
400	# if defined(VBOX_INCLUDED_vmm_dbgf_h) \|\| defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
401
402	/**
403	* Initializes the execution state.
404	*
405	* @param pVCpu The cross context virtual CPU structure of the
406	* calling thread.
407	* @param fExecOpts Optional execution flags:
408	* - IEM_F_BYPASS_HANDLERS
409	* - IEM_F_X86_DISREGARD_LOCK
410	*
411	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
412	* side-effects in strict builds.
413	*/
414	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
415	{
416	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
417	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
418	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
419	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
420	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
421	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
422	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
423	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
424	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
425	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
426
427	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
428	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| fExecOpts;
429	pVCpu->iem.s.cActiveMappings = 0;
430	pVCpu->iem.s.iNextMapping = 0;
431
432	# ifdef VBOX_STRICT
433	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
434	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
435	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
436	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
437	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
438	pVCpu->iem.s.uRexReg = 127;
439	pVCpu->iem.s.uRexB = 127;
440	pVCpu->iem.s.offModRm = 127;
441	pVCpu->iem.s.uRexIndex = 127;
442	pVCpu->iem.s.iEffSeg = 127;
443	pVCpu->iem.s.idxPrefix = 127;
444	pVCpu->iem.s.uVex3rdReg = 127;
445	pVCpu->iem.s.uVexLength = 127;
446	pVCpu->iem.s.fEvexStuff = 127;
447	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
448	# ifdef IEM_WITH_CODE_TLB
449	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
450	pVCpu->iem.s.pbInstrBuf = NULL;
451	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
452	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
453	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
454	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
455	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
456	pVCpu->iem.s.offOpcode = 127;
457	# endif
458	# else
459	pVCpu->iem.s.offOpcode = 127;
460	pVCpu->iem.s.cbOpcode = 127;
461	# endif
462	# endif /* VBOX_STRICT */
463	}
464
465
466	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
467	/**
468	* Performs a minimal reinitialization of the execution state.
469	*
470	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
471	* 'world-switch' types operations on the CPU. Currently only nested
472	* hardware-virtualization uses it.
473	*
474	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
475	* @param cbInstr The instruction length (for flushing).
476	*/
477	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
478	{
479	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
480	iemOpcodeFlushHeavy(pVCpu, cbInstr);
481	}
482	# endif
483
484	# endif /* VBOX_INCLUDED_vmm_dbgf_h \|\| DOXYGEN_RUNNING */
485
486	/**
487	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
488	*
489	* @param pVCpu The cross context virtual CPU structure of the
490	* calling thread.
491	*/
492	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
493	{
494	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
495	# ifdef VBOX_STRICT
496	# ifdef IEM_WITH_CODE_TLB
497	NOREF(pVCpu);
498	# else
499	pVCpu->iem.s.cbOpcode = 0;
500	# endif
501	# else
502	NOREF(pVCpu);
503	# endif
504	}
505
506
507	/**
508	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
509	*
510	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
511	*
512	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
513	* @param pVCpu The cross context virtual CPU structure of the calling thread.
514	* @param rcStrict The status code to fiddle.
515	*/
516	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
517	{
518	iemUninitExec(pVCpu);
519	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
520	}
521
522
523	/**
524	* Macro used by the IEMExec* method to check the given instruction length.
525	*
526	* Will return on failure!
527	*
528	* @param a_cbInstr The given instruction length.
529	* @param a_cbMin The minimum length.
530	*/
531	# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
532	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
533	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
534
535
536	# ifndef IEM_WITH_SETJMP
537
538	/**
539	* Fetches the first opcode byte.
540	*
541	* @returns Strict VBox status code.
542	* @param pVCpu The cross context virtual CPU structure of the
543	* calling thread.
544	* @param pu8 Where to return the opcode byte.
545	*/
546	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
547	{
548	/*
549	* Check for hardware instruction breakpoints.
550	*/
551	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
552	{ /* likely */ }
553	else
554	{
555	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
556	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
557	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
558	{ /* likely */ }
559	else
560	{
561	pu8 = 0xff; / shut up gcc. sigh */
562	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
563	return iemRaiseDebugException(pVCpu);
564	return rcStrict;
565	}
566	}
567
568	/*
569	* Fetch the first opcode byte.
570	*/
571	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
572	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
573	{
574	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
575	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
576	return VINF_SUCCESS;
577	}
578	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
579	}
580
581	# else /* IEM_WITH_SETJMP */
582
583	/**
584	* Fetches the first opcode byte, longjmp on error.
585	*
586	* @returns The opcode byte.
587	* @param pVCpu The cross context virtual CPU structure of the calling thread.
588	*/
589	DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
590	{
591	/*
592	* Check for hardware instruction breakpoints.
593	*/
594	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
595	{ /* likely */ }
596	else
597	{
598	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
599	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
600	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
601	{ /* likely */ }
602	else
603	{
604	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
605	rcStrict = iemRaiseDebugException(pVCpu);
606	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
607	}
608	}
609
610	/*
611	* Fetch the first opcode byte.
612	*/
613	# ifdef IEM_WITH_CODE_TLB
614	uint8_t bRet;
615	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
616	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
617	if (RT_LIKELY( pbBuf != NULL
618	&& offBuf < pVCpu->iem.s.cbInstrBuf))
619	{
620	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
621	bRet = pbBuf[offBuf];
622	}
623	else
624	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
625	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
626	Assert(pVCpu->iem.s.offOpcode == 0);
627	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
628	# endif
629	return bRet;
630
631	# else /* !IEM_WITH_CODE_TLB */
632	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
633	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
634	{
635	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
636	return pVCpu->iem.s.abOpcode[offOpcode];
637	}
638	return iemOpcodeGetNextU8SlowJmp(pVCpu);
639	# endif
640	}
641
642	# endif /* IEM_WITH_SETJMP */
643
644	/**
645	* Fetches the first opcode byte, returns/throws automatically on failure.
646	*
647	* @param a_pu8 Where to return the opcode byte.
648	* @remark Implicitly references pVCpu.
649	*/
650	# ifndef IEM_WITH_SETJMP
651	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
652	do \
653	{ \
654	VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
655	if (rcStrict2 == VINF_SUCCESS) \
656	{ /* likely */ } \
657	else \
658	return rcStrict2; \
659	} while (0)
660	# else
661	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
662	# endif /* IEM_WITH_SETJMP */
663
664
665	# ifndef IEM_WITH_SETJMP
666
667	/**
668	* Fetches the next opcode byte.
669	*
670	* @returns Strict VBox status code.
671	* @param pVCpu The cross context virtual CPU structure of the
672	* calling thread.
673	* @param pu8 Where to return the opcode byte.
674	*/
675	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
676	{
677	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
678	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
679	{
680	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
681	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
682	return VINF_SUCCESS;
683	}
684	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
685	}
686
687	# else /* IEM_WITH_SETJMP */
688
689	/**
690	* Fetches the next opcode byte, longjmp on error.
691	*
692	* @returns The opcode byte.
693	* @param pVCpu The cross context virtual CPU structure of the calling thread.
694	*/
695	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
696	{
697	# ifdef IEM_WITH_CODE_TLB
698	uint8_t bRet;
699	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
700	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
701	if (RT_LIKELY( pbBuf != NULL
702	&& offBuf < pVCpu->iem.s.cbInstrBuf))
703	{
704	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
705	bRet = pbBuf[offBuf];
706	}
707	else
708	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
709	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
710	Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode));
711	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
712	# endif
713	return bRet;
714
715	# else /* !IEM_WITH_CODE_TLB */
716	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
717	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
718	{
719	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
720	return pVCpu->iem.s.abOpcode[offOpcode];
721	}
722	return iemOpcodeGetNextU8SlowJmp(pVCpu);
723	# endif
724	}
725
726	# endif /* IEM_WITH_SETJMP */
727
728	/**
729	* Fetches the next opcode byte, returns automatically on failure.
730	*
731	* @param a_pu8 Where to return the opcode byte.
732	* @remark Implicitly references pVCpu.
733	*/
734	# ifndef IEM_WITH_SETJMP
735	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
736	do \
737	{ \
738	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
739	if (rcStrict2 == VINF_SUCCESS) \
740	{ /* likely */ } \
741	else \
742	return rcStrict2; \
743	} while (0)
744	# else
745	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
746	# endif /* IEM_WITH_SETJMP */
747
748
749	# ifndef IEM_WITH_SETJMP
750	/**
751	* Fetches the next signed byte from the opcode stream.
752	*
753	* @returns Strict VBox status code.
754	* @param pVCpu The cross context virtual CPU structure of the calling thread.
755	* @param pi8 Where to return the signed byte.
756	*/
757	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
758	{
759	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
760	}
761	# endif /* !IEM_WITH_SETJMP */
762
763
764	/**
765	* Fetches the next signed byte from the opcode stream, returning automatically
766	* on failure.
767	*
768	* @param a_pi8 Where to return the signed byte.
769	* @remark Implicitly references pVCpu.
770	*/
771	# ifndef IEM_WITH_SETJMP
772	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
773	do \
774	{ \
775	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
776	if (rcStrict2 != VINF_SUCCESS) \
777	return rcStrict2; \
778	} while (0)
779	# else /* IEM_WITH_SETJMP */
780	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
781
782	# endif /* IEM_WITH_SETJMP */
783
784
785	# ifndef IEM_WITH_SETJMP
786	/**
787	* Fetches the next signed byte from the opcode stream, extending it to
788	* unsigned 16-bit.
789	*
790	* @returns Strict VBox status code.
791	* @param pVCpu The cross context virtual CPU structure of the calling thread.
792	* @param pu16 Where to return the unsigned word.
793	*/
794	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
795	{
796	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
797	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
798	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
799
800	*pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
801	pVCpu->iem.s.offOpcode = offOpcode + 1;
802	return VINF_SUCCESS;
803	}
804	# endif /* !IEM_WITH_SETJMP */
805
806	/**
807	* Fetches the next signed byte from the opcode stream and sign-extending it to
808	* a word, returning automatically on failure.
809	*
810	* @param a_pu16 Where to return the word.
811	* @remark Implicitly references pVCpu.
812	*/
813	# ifndef IEM_WITH_SETJMP
814	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
815	do \
816	{ \
817	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
818	if (rcStrict2 != VINF_SUCCESS) \
819	return rcStrict2; \
820	} while (0)
821	# else
822	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
823	# endif
824
825	# ifndef IEM_WITH_SETJMP
826	/**
827	* Fetches the next signed byte from the opcode stream, extending it to
828	* unsigned 32-bit.
829	*
830	* @returns Strict VBox status code.
831	* @param pVCpu The cross context virtual CPU structure of the calling thread.
832	* @param pu32 Where to return the unsigned dword.
833	*/
834	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
835	{
836	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
837	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
838	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
839
840	*pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
841	pVCpu->iem.s.offOpcode = offOpcode + 1;
842	return VINF_SUCCESS;
843	}
844	# endif /* !IEM_WITH_SETJMP */
845
846	/**
847	* Fetches the next signed byte from the opcode stream and sign-extending it to
848	* a word, returning automatically on failure.
849	*
850	* @param a_pu32 Where to return the word.
851	* @remark Implicitly references pVCpu.
852	*/
853	# ifndef IEM_WITH_SETJMP
854	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
855	do \
856	{ \
857	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
858	if (rcStrict2 != VINF_SUCCESS) \
859	return rcStrict2; \
860	} while (0)
861	# else
862	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
863	# endif
864
865
866	# ifndef IEM_WITH_SETJMP
867	/**
868	* Fetches the next signed byte from the opcode stream, extending it to
869	* unsigned 64-bit.
870	*
871	* @returns Strict VBox status code.
872	* @param pVCpu The cross context virtual CPU structure of the calling thread.
873	* @param pu64 Where to return the unsigned qword.
874	*/
875	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
876	{
877	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
878	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
879	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
880
881	*pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
882	pVCpu->iem.s.offOpcode = offOpcode + 1;
883	return VINF_SUCCESS;
884	}
885	# endif /* !IEM_WITH_SETJMP */
886
887	/**
888	* Fetches the next signed byte from the opcode stream and sign-extending it to
889	* a word, returning automatically on failure.
890	*
891	* @param a_pu64 Where to return the word.
892	* @remark Implicitly references pVCpu.
893	*/
894	# ifndef IEM_WITH_SETJMP
895	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
896	do \
897	{ \
898	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
899	if (rcStrict2 != VINF_SUCCESS) \
900	return rcStrict2; \
901	} while (0)
902	# else
903	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
904	# endif
905
906
907	# ifndef IEM_WITH_SETJMP
908
909	/**
910	* Fetches the next opcode word.
911	*
912	* @returns Strict VBox status code.
913	* @param pVCpu The cross context virtual CPU structure of the calling thread.
914	* @param pu16 Where to return the opcode word.
915	*/
916	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
917	{
918	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
919	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
920	{
921	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
922	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
923	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
924	# else
925	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
926	# endif
927	return VINF_SUCCESS;
928	}
929	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
930	}
931
932	# else /* IEM_WITH_SETJMP */
933
934	/**
935	* Fetches the next opcode word, longjmp on error.
936	*
937	* @returns The opcode word.
938	* @param pVCpu The cross context virtual CPU structure of the calling thread.
939	*/
940	DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
941	{
942	# ifdef IEM_WITH_CODE_TLB
943	uint16_t u16Ret;
944	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
945	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
946	if (RT_LIKELY( pbBuf != NULL
947	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
948	{
949	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
950	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
951	u16Ret = (uint16_t const )&pbBuf[offBuf];
952	# else
953	u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
954	# endif
955	}
956	else
957	u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu);
958
959	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
960	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
961	Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode));
962	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
963	(uint16_t )&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret;
964	# else
965	pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret);
966	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret);
967	# endif
968	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2;
969	# endif
970
971	return u16Ret;
972
973	# else /* !IEM_WITH_CODE_TLB */
974	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
975	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
976	{
977	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
978	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
979	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
980	# else
981	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
982	# endif
983	}
984	return iemOpcodeGetNextU16SlowJmp(pVCpu);
985	# endif /* !IEM_WITH_CODE_TLB */
986	}
987
988	# endif /* IEM_WITH_SETJMP */
989
990	/**
991	* Fetches the next opcode word, returns automatically on failure.
992	*
993	* @param a_pu16 Where to return the opcode word.
994	* @remark Implicitly references pVCpu.
995	*/
996	# ifndef IEM_WITH_SETJMP
997	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
998	do \
999	{ \
1000	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
1001	if (rcStrict2 != VINF_SUCCESS) \
1002	return rcStrict2; \
1003	} while (0)
1004	# else
1005	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
1006	# endif
1007
1008	# ifndef IEM_WITH_SETJMP
1009	/**
1010	* Fetches the next opcode word, zero extending it to a double word.
1011	*
1012	* @returns Strict VBox status code.
1013	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1014	* @param pu32 Where to return the opcode double word.
1015	*/
1016	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1017	{
1018	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1019	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1020	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
1021
1022	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1023	pVCpu->iem.s.offOpcode = offOpcode + 2;
1024	return VINF_SUCCESS;
1025	}
1026	# endif /* !IEM_WITH_SETJMP */
1027
1028	/**
1029	* Fetches the next opcode word and zero extends it to a double word, returns
1030	* automatically on failure.
1031	*
1032	* @param a_pu32 Where to return the opcode double word.
1033	* @remark Implicitly references pVCpu.
1034	*/
1035	# ifndef IEM_WITH_SETJMP
1036	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1037	do \
1038	{ \
1039	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
1040	if (rcStrict2 != VINF_SUCCESS) \
1041	return rcStrict2; \
1042	} while (0)
1043	# else
1044	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
1045	# endif
1046
1047	# ifndef IEM_WITH_SETJMP
1048	/**
1049	* Fetches the next opcode word, zero extending it to a quad word.
1050	*
1051	* @returns Strict VBox status code.
1052	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1053	* @param pu64 Where to return the opcode quad word.
1054	*/
1055	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1056	{
1057	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1058	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1059	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
1060
1061	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1062	pVCpu->iem.s.offOpcode = offOpcode + 2;
1063	return VINF_SUCCESS;
1064	}
1065	# endif /* !IEM_WITH_SETJMP */
1066
1067	/**
1068	* Fetches the next opcode word and zero extends it to a quad word, returns
1069	* automatically on failure.
1070	*
1071	* @param a_pu64 Where to return the opcode quad word.
1072	* @remark Implicitly references pVCpu.
1073	*/
1074	# ifndef IEM_WITH_SETJMP
1075	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1076	do \
1077	{ \
1078	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
1079	if (rcStrict2 != VINF_SUCCESS) \
1080	return rcStrict2; \
1081	} while (0)
1082	# else
1083	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
1084	# endif
1085
1086
1087	# ifndef IEM_WITH_SETJMP
1088	/**
1089	* Fetches the next signed word from the opcode stream.
1090	*
1091	* @returns Strict VBox status code.
1092	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1093	* @param pi16 Where to return the signed word.
1094	*/
1095	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
1096	{
1097	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
1098	}
1099	# endif /* !IEM_WITH_SETJMP */
1100
1101
1102	/**
1103	* Fetches the next signed word from the opcode stream, returning automatically
1104	* on failure.
1105	*
1106	* @param a_pi16 Where to return the signed word.
1107	* @remark Implicitly references pVCpu.
1108	*/
1109	# ifndef IEM_WITH_SETJMP
1110	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1111	do \
1112	{ \
1113	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
1114	if (rcStrict2 != VINF_SUCCESS) \
1115	return rcStrict2; \
1116	} while (0)
1117	# else
1118	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
1119	# endif
1120
1121	# ifndef IEM_WITH_SETJMP
1122
1123	/**
1124	* Fetches the next opcode dword.
1125	*
1126	* @returns Strict VBox status code.
1127	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1128	* @param pu32 Where to return the opcode double word.
1129	*/
1130	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1131	{
1132	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1133	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1134	{
1135	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1136	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1137	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1138	# else
1139	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1140	pVCpu->iem.s.abOpcode[offOpcode + 1],
1141	pVCpu->iem.s.abOpcode[offOpcode + 2],
1142	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1143	# endif
1144	return VINF_SUCCESS;
1145	}
1146	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
1147	}
1148
1149	# else /* IEM_WITH_SETJMP */
1150
1151	/**
1152	* Fetches the next opcode dword, longjmp on error.
1153	*
1154	* @returns The opcode dword.
1155	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1156	*/
1157	DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1158	{
1159	# ifdef IEM_WITH_CODE_TLB
1160	uint32_t u32Ret;
1161	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1162	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1163	if (RT_LIKELY( pbBuf != NULL
1164	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1165	{
1166	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1167	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1168	u32Ret = (uint32_t const )&pbBuf[offBuf];
1169	# else
1170	u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1171	pbBuf[offBuf + 1],
1172	pbBuf[offBuf + 2],
1173	pbBuf[offBuf + 3]);
1174	# endif
1175	}
1176	else
1177	u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu);
1178
1179	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1180	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1181	Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode));
1182	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1183	(uint32_t )&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret;
1184	# else
1185	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret);
1186	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret);
1187	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret);
1188	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret);
1189	# endif
1190	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4;
1191	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1192
1193	return u32Ret;
1194
1195	# else /* !IEM_WITH_CODE_TLB */
1196	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1197	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1198	{
1199	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1200	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1201	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1202	# else
1203	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1204	pVCpu->iem.s.abOpcode[offOpcode + 1],
1205	pVCpu->iem.s.abOpcode[offOpcode + 2],
1206	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1207	# endif
1208	}
1209	return iemOpcodeGetNextU32SlowJmp(pVCpu);
1210	# endif
1211	}
1212
1213	# endif /* IEM_WITH_SETJMP */
1214
1215	/**
1216	* Fetches the next opcode dword, returns automatically on failure.
1217	*
1218	* @param a_pu32 Where to return the opcode dword.
1219	* @remark Implicitly references pVCpu.
1220	*/
1221	# ifndef IEM_WITH_SETJMP
1222	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1223	do \
1224	{ \
1225	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1226	if (rcStrict2 != VINF_SUCCESS) \
1227	return rcStrict2; \
1228	} while (0)
1229	# else
1230	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1231	# endif
1232
1233	# ifndef IEM_WITH_SETJMP
1234	/**
1235	* Fetches the next opcode dword, zero extending it to a quad word.
1236	*
1237	* @returns Strict VBox status code.
1238	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1239	* @param pu64 Where to return the opcode quad word.
1240	*/
1241	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1242	{
1243	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1244	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1245	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1246
1247	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1248	pVCpu->iem.s.abOpcode[offOpcode + 1],
1249	pVCpu->iem.s.abOpcode[offOpcode + 2],
1250	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1251	pVCpu->iem.s.offOpcode = offOpcode + 4;
1252	return VINF_SUCCESS;
1253	}
1254	# endif /* !IEM_WITH_SETJMP */
1255
1256	/**
1257	* Fetches the next opcode dword and zero extends it to a quad word, returns
1258	* automatically on failure.
1259	*
1260	* @param a_pu64 Where to return the opcode quad word.
1261	* @remark Implicitly references pVCpu.
1262	*/
1263	# ifndef IEM_WITH_SETJMP
1264	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1265	do \
1266	{ \
1267	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1268	if (rcStrict2 != VINF_SUCCESS) \
1269	return rcStrict2; \
1270	} while (0)
1271	# else
1272	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1273	# endif
1274
1275
1276	# ifndef IEM_WITH_SETJMP
1277	/**
1278	* Fetches the next signed double word from the opcode stream.
1279	*
1280	* @returns Strict VBox status code.
1281	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1282	* @param pi32 Where to return the signed double word.
1283	*/
1284	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1285	{
1286	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1287	}
1288	# endif
1289
1290	/**
1291	* Fetches the next signed double word from the opcode stream, returning
1292	* automatically on failure.
1293	*
1294	* @param a_pi32 Where to return the signed double word.
1295	* @remark Implicitly references pVCpu.
1296	*/
1297	# ifndef IEM_WITH_SETJMP
1298	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1299	do \
1300	{ \
1301	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1302	if (rcStrict2 != VINF_SUCCESS) \
1303	return rcStrict2; \
1304	} while (0)
1305	# else
1306	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1307	# endif
1308
1309	# ifndef IEM_WITH_SETJMP
1310	/**
1311	* Fetches the next opcode dword, sign extending it into a quad word.
1312	*
1313	* @returns Strict VBox status code.
1314	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1315	* @param pu64 Where to return the opcode quad word.
1316	*/
1317	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1318	{
1319	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1320	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1321	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1322
1323	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1324	pVCpu->iem.s.abOpcode[offOpcode + 1],
1325	pVCpu->iem.s.abOpcode[offOpcode + 2],
1326	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1327	*pu64 = (uint64_t)(int64_t)i32;
1328	pVCpu->iem.s.offOpcode = offOpcode + 4;
1329	return VINF_SUCCESS;
1330	}
1331	# endif /* !IEM_WITH_SETJMP */
1332
1333	/**
1334	* Fetches the next opcode double word and sign extends it to a quad word,
1335	* returns automatically on failure.
1336	*
1337	* @param a_pu64 Where to return the opcode quad word.
1338	* @remark Implicitly references pVCpu.
1339	*/
1340	# ifndef IEM_WITH_SETJMP
1341	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1342	do \
1343	{ \
1344	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1345	if (rcStrict2 != VINF_SUCCESS) \
1346	return rcStrict2; \
1347	} while (0)
1348	# else
1349	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1350	# endif
1351
1352	# ifndef IEM_WITH_SETJMP
1353
1354	/**
1355	* Fetches the next opcode qword.
1356	*
1357	* @returns Strict VBox status code.
1358	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1359	* @param pu64 Where to return the opcode qword.
1360	*/
1361	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1362	{
1363	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1364	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1365	{
1366	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1367	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1368	# else
1369	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1370	pVCpu->iem.s.abOpcode[offOpcode + 1],
1371	pVCpu->iem.s.abOpcode[offOpcode + 2],
1372	pVCpu->iem.s.abOpcode[offOpcode + 3],
1373	pVCpu->iem.s.abOpcode[offOpcode + 4],
1374	pVCpu->iem.s.abOpcode[offOpcode + 5],
1375	pVCpu->iem.s.abOpcode[offOpcode + 6],
1376	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1377	# endif
1378	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1379	return VINF_SUCCESS;
1380	}
1381	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1382	}
1383
1384	# else /* IEM_WITH_SETJMP */
1385
1386	/**
1387	* Fetches the next opcode qword, longjmp on error.
1388	*
1389	* @returns The opcode qword.
1390	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1391	*/
1392	DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1393	{
1394	# ifdef IEM_WITH_CODE_TLB
1395	uint64_t u64Ret;
1396	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1397	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1398	if (RT_LIKELY( pbBuf != NULL
1399	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1400	{
1401	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1402	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1403	u64Ret = (uint64_t const )&pbBuf[offBuf];
1404	# else
1405	u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1406	pbBuf[offBuf + 1],
1407	pbBuf[offBuf + 2],
1408	pbBuf[offBuf + 3],
1409	pbBuf[offBuf + 4],
1410	pbBuf[offBuf + 5],
1411	pbBuf[offBuf + 6],
1412	pbBuf[offBuf + 7]);
1413	# endif
1414	}
1415	else
1416	u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu);
1417
1418	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1419	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1420	Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode));
1421	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1422	(uint64_t )&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret;
1423	# else
1424	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret);
1425	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret);
1426	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret);
1427	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret);
1428	pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret);
1429	pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret);
1430	pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret);
1431	pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret);
1432	# endif
1433	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8;
1434	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1435
1436	return u64Ret;
1437
1438	# else /* !IEM_WITH_CODE_TLB */
1439	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1440	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1441	{
1442	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1443	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1444	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1445	# else
1446	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1447	pVCpu->iem.s.abOpcode[offOpcode + 1],
1448	pVCpu->iem.s.abOpcode[offOpcode + 2],
1449	pVCpu->iem.s.abOpcode[offOpcode + 3],
1450	pVCpu->iem.s.abOpcode[offOpcode + 4],
1451	pVCpu->iem.s.abOpcode[offOpcode + 5],
1452	pVCpu->iem.s.abOpcode[offOpcode + 6],
1453	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1454	# endif
1455	}
1456	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1457	# endif /* !IEM_WITH_CODE_TLB */
1458	}
1459
1460	# endif /* IEM_WITH_SETJMP */
1461
1462	/**
1463	* Fetches the next opcode quad word, returns automatically on failure.
1464	*
1465	* @param a_pu64 Where to return the opcode quad word.
1466	* @remark Implicitly references pVCpu.
1467	*/
1468	# ifndef IEM_WITH_SETJMP
1469	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1470	do \
1471	{ \
1472	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1473	if (rcStrict2 != VINF_SUCCESS) \
1474	return rcStrict2; \
1475	} while (0)
1476	# else
1477	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1478	# endif
1479
1480	/**
1481	* For fetching the opcode bytes for an ModR/M effective address, but throw
1482	* away the result.
1483	*
1484	* This is used when decoding undefined opcodes and such where we want to avoid
1485	* unnecessary MC blocks.
1486	*
1487	* @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is
1488	* used instead. At least for now...
1489	*/
1490	# ifndef IEM_WITH_SETJMP
1491	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1492	RTGCPTR GCPtrEff; \
1493	VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \
1494	if (rcStrict != VINF_SUCCESS) \
1495	return rcStrict; \
1496	} while (0)
1497	# else
1498	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1499	(void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \
1500	} while (0)
1501	# endif
1502
1503	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1504
1505
1506	/** @name Misc Worker Functions.
1507	* @{
1508	*/
1509
1510	/**
1511	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1512	* not (kind of obsolete now).
1513	*
1514	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1515	*/
1516	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1517
1518	/**
1519	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1520	*
1521	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1522	* @param a_fEfl The new EFLAGS.
1523	*/
1524	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1525
1526
1527	/**
1528	* Loads a NULL data selector into a selector register, both the hidden and
1529	* visible parts, in protected mode.
1530	*
1531	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1532	* @param pSReg Pointer to the segment register.
1533	* @param uRpl The RPL.
1534	*/
1535	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1536	{
1537	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1538	* data selector in protected mode. */
1539	pSReg->Sel = uRpl;
1540	pSReg->ValidSel = uRpl;
1541	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1542	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1543	{
1544	/* VT-x (Intel 3960x) observed doing something like this. */
1545	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);
1546	pSReg->u32Limit = UINT32_MAX;
1547	pSReg->u64Base = 0;
1548	}
1549	else
1550	{
1551	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1552	pSReg->u32Limit = 0;
1553	pSReg->u64Base = 0;
1554	}
1555	}
1556
1557	/** @} */
1558
1559
1560	/*
1561	*
1562	* Helpers routines.
1563	* Helpers routines.
1564	* Helpers routines.
1565	*
1566	*/
1567
1568	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1569
1570	/**
1571	* Recalculates the effective operand size.
1572	*
1573	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1574	*/
1575	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1576	{
1577	switch (IEM_GET_CPU_MODE(pVCpu))
1578	{
1579	case IEMMODE_16BIT:
1580	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1581	break;
1582	case IEMMODE_32BIT:
1583	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1584	break;
1585	case IEMMODE_64BIT:
1586	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1587	{
1588	case 0:
1589	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1590	break;
1591	case IEM_OP_PRF_SIZE_OP:
1592	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1593	break;
1594	case IEM_OP_PRF_SIZE_REX_W:
1595	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1596	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1597	break;
1598	}
1599	break;
1600	default:
1601	AssertFailed();
1602	}
1603	}
1604
1605
1606	/**
1607	* Sets the default operand size to 64-bit and recalculates the effective
1608	* operand size.
1609	*
1610	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1611	*/
1612	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1613	{
1614	Assert(IEM_IS_64BIT_CODE(pVCpu));
1615	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1616	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1617	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1618	else
1619	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1620	}
1621
1622
1623	/**
1624	* Sets the default operand size to 64-bit and recalculates the effective
1625	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1626	*
1627	* This is for the relative jumps.
1628	*
1629	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1630	*/
1631	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1632	{
1633	Assert(IEM_IS_64BIT_CODE(pVCpu));
1634	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1635	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1636	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1637	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1638	else
1639	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1640	}
1641
1642	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1643
1644
1645
1646	/** @name Register Access.
1647	* @{
1648	*/
1649
1650	/**
1651	* Gets a reference (pointer) to the specified hidden segment register.
1652	*
1653	* @returns Hidden register reference.
1654	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1655	* @param iSegReg The segment register.
1656	*/
1657	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1658	{
1659	Assert(iSegReg < X86_SREG_COUNT);
1660	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1661	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1662
1663	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1664	return pSReg;
1665	}
1666
1667
1668	/**
1669	* Ensures that the given hidden segment register is up to date.
1670	*
1671	* @returns Hidden register reference.
1672	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1673	* @param pSReg The segment register.
1674	*/
1675	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1676	{
1677	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1678	NOREF(pVCpu);
1679	return pSReg;
1680	}
1681
1682
1683	/**
1684	* Gets a reference (pointer) to the specified segment register (the selector
1685	* value).
1686	*
1687	* @returns Pointer to the selector variable.
1688	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1689	* @param iSegReg The segment register.
1690	*/
1691	DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1692	{
1693	Assert(iSegReg < X86_SREG_COUNT);
1694	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1695	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1696	}
1697
1698
1699	/**
1700	* Fetches the selector value of a segment register.
1701	*
1702	* @returns The selector value.
1703	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1704	* @param iSegReg The segment register.
1705	*/
1706	DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1707	{
1708	Assert(iSegReg < X86_SREG_COUNT);
1709	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1710	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1711	}
1712
1713
1714	/**
1715	* Fetches the base address value of a segment register.
1716	*
1717	* @returns The selector value.
1718	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1719	* @param iSegReg The segment register.
1720	*/
1721	DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1722	{
1723	Assert(iSegReg < X86_SREG_COUNT);
1724	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1725	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1726	}
1727
1728
1729	/**
1730	* Gets a reference (pointer) to the specified general purpose register.
1731	*
1732	* @returns Register reference.
1733	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1734	* @param iReg The general purpose register.
1735	*/
1736	DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1737	{
1738	Assert(iReg < 16);
1739	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1740	}
1741
1742
1743	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1744	/**
1745	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1746	*
1747	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1748	*
1749	* @returns Register reference.
1750	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1751	* @param iReg The register.
1752	*/
1753	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1754	{
1755	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1756	{
1757	Assert(iReg < 16);
1758	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1759	}
1760	/* high 8-bit register. */
1761	Assert(iReg < 8);
1762	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1763	}
1764	#endif
1765
1766
1767	/**
1768	* Gets a reference (pointer) to the specified 8-bit general purpose register,
1769	* alternative version with extended (20) register index.
1770	*
1771	* @returns Register reference.
1772	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1773	* @param iRegEx The register. The 16 first are regular ones,
1774	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1775	*/
1776	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1777	{
1778	/** @todo This could be done by double indexing on little endian hosts:
1779	* return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */
1780	if (iRegEx < 16)
1781	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1782
1783	/* high 8-bit register. */
1784	Assert(iRegEx < 20);
1785	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1786	}
1787
1788
1789	/**
1790	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1791	*
1792	* @returns Register reference.
1793	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1794	* @param iReg The register.
1795	*/
1796	DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1797	{
1798	Assert(iReg < 16);
1799	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1800	}
1801
1802
1803	/**
1804	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1805	*
1806	* @returns Register reference.
1807	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1808	* @param iReg The register.
1809	*/
1810	DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1811	{
1812	Assert(iReg < 16);
1813	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1814	}
1815
1816
1817	/**
1818	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1819	*
1820	* @returns Register reference.
1821	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1822	* @param iReg The register.
1823	*/
1824	DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1825	{
1826	Assert(iReg < 16);
1827	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1828	}
1829
1830
1831	/**
1832	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1833	*
1834	* @returns Register reference.
1835	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1836	* @param iReg The register.
1837	*/
1838	DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1839	{
1840	Assert(iReg < 64);
1841	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1842	}
1843
1844
1845	/**
1846	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1847	*
1848	* @returns Register reference.
1849	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1850	* @param iReg The register.
1851	*/
1852	DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1853	{
1854	Assert(iReg < 16);
1855	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1856	}
1857
1858
1859	/**
1860	* Gets a reference (pointer) to the specified segment register's base address.
1861	*
1862	* @returns Segment register base address reference.
1863	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1864	* @param iSegReg The segment selector.
1865	*/
1866	DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1867	{
1868	Assert(iSegReg < X86_SREG_COUNT);
1869	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1870	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1871	}
1872
1873
1874	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1875	/**
1876	* Fetches the value of a 8-bit general purpose register.
1877	*
1878	* @returns The register value.
1879	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1880	* @param iReg The register.
1881	*/
1882	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1883	{
1884	return *iemGRegRefU8(pVCpu, iReg);
1885	}
1886	#endif
1887
1888
1889	/**
1890	* Fetches the value of a 8-bit general purpose register, alternative version
1891	* with extended (20) register index.
1892
1893	* @returns The register value.
1894	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1895	* @param iRegEx The register. The 16 first are regular ones,
1896	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1897	*/
1898	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1899	{
1900	return *iemGRegRefU8Ex(pVCpu, iRegEx);
1901	}
1902
1903
1904	/**
1905	* Fetches the value of a 16-bit general purpose register.
1906	*
1907	* @returns The register value.
1908	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1909	* @param iReg The register.
1910	*/
1911	DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1912	{
1913	Assert(iReg < 16);
1914	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1915	}
1916
1917
1918	/**
1919	* Fetches the value of a 32-bit general purpose register.
1920	*
1921	* @returns The register value.
1922	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1923	* @param iReg The register.
1924	*/
1925	DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1926	{
1927	Assert(iReg < 16);
1928	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1929	}
1930
1931
1932	/**
1933	* Fetches the value of a 64-bit general purpose register.
1934	*
1935	* @returns The register value.
1936	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1937	* @param iReg The register.
1938	*/
1939	DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1940	{
1941	Assert(iReg < 16);
1942	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1943	}
1944
1945
1946	/**
1947	* Stores a 16-bit value to a general purpose register.
1948	*
1949	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1950	* @param iReg The register.
1951	* @param uValue The value to store.
1952	*/
1953	DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT
1954	{
1955	Assert(iReg < 16);
1956	pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;
1957	}
1958
1959
1960	/**
1961	* Stores a 32-bit value to a general purpose register, implicitly clearing high
1962	* values.
1963	*
1964	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1965	* @param iReg The register.
1966	* @param uValue The value to store.
1967	*/
1968	DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT
1969	{
1970	Assert(iReg < 16);
1971	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1972	}
1973
1974
1975	/**
1976	* Stores a 64-bit value to a general purpose register.
1977	*
1978	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1979	* @param iReg The register.
1980	* @param uValue The value to store.
1981	*/
1982	DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT
1983	{
1984	Assert(iReg < 16);
1985	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1986	}
1987
1988
1989	/**
1990	* Get the address of the top of the stack.
1991	*
1992	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1993	*/
1994	DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1995	{
1996	if (IEM_IS_64BIT_CODE(pVCpu))
1997	return pVCpu->cpum.GstCtx.rsp;
1998	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1999	return pVCpu->cpum.GstCtx.esp;
2000	return pVCpu->cpum.GstCtx.sp;
2001	}
2002
2003
2004	/**
2005	* Updates the RIP/EIP/IP to point to the next instruction.
2006	*
2007	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2008	* @param cbInstr The number of bytes to add.
2009	*/
2010	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2011	{
2012	/*
2013	* Advance RIP.
2014	*
2015	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
2016	* while in all other modes except LM64 the updates are 32-bit. This means
2017	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
2018	* 4GB and 64KB rollovers, and decide whether anything needs masking.
2019	*
2020	* See PC wrap around tests in bs3-cpu-weird-1.
2021	*/
2022	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
2023	uint64_t const uRipNext = uRipPrev + cbInstr;
2024	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
2025	\|\| IEM_IS_64BIT_CODE(pVCpu)))
2026	pVCpu->cpum.GstCtx.rip = uRipNext;
2027	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
2028	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
2029	else
2030	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
2031	}
2032
2033
2034	/**
2035	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
2036	* following EFLAGS bits are set:
2037	* - X86_EFL_RF - clear it.
2038	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
2039	* - X86_EFL_TF - generate single step \#DB trap.
2040	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
2041	* instruction).
2042	*
2043	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
2044	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
2045	* takes priority over both NMIs and hardware interrupts. So, neither is
2046	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
2047	* either unsupported will be triggered on-top of any \#DB raised here.)
2048	*
2049	* The RF flag only needs to be cleared here as it only suppresses instruction
2050	* breakpoints which are not raised here (happens synchronously during
2051	* instruction fetching).
2052	*
2053	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
2054	* status has no bearing on whether \#DB exceptions are raised.
2055	*
2056	* @note This must NOT be called by the two instructions setting the
2057	* CPUMCTX_INHIBIT_SHADOW_SS flag.
2058	*
2059	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
2060	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
2061	* Stacks}
2062	*/
2063	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2064	{
2065	/*
2066	* Normally we're just here to clear RF and/or interrupt shadow bits.
2067	*/
2068	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) == 0))
2069	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
2070	else
2071	{
2072	/*
2073	* Raise a #DB or/and DBGF event.
2074	*/
2075	VBOXSTRICTRC rcStrict;
2076	if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK))
2077	{
2078	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2079	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2080	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
2081	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
2082	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2083	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
2084	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2085	pVCpu->cpum.GstCtx.rflags.uBoth));
2086
2087	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
2088	rcStrict = iemRaiseDebugException(pVCpu);
2089
2090	/* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
2091	if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
2092	{
2093	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2094	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2095	}
2096	}
2097	else
2098	{
2099	Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
2100	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2101	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2102	}
2103	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
2104	return rcStrict;
2105	}
2106	return VINF_SUCCESS;
2107	}
2108
2109
2110	/**
2111	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
2112	*
2113	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2114	*/
2115	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2116	{
2117	/*
2118	* We assume that most of the time nothing actually needs doing here.
2119	*/
2120	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2121	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
2122	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
2123	return VINF_SUCCESS;
2124	return iemFinishInstructionWithFlagsSet(pVCpu);
2125	}
2126
2127
2128	/**
2129	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
2130	* and CPUMCTX_INHIBIT_SHADOW.
2131	*
2132	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2133	* @param cbInstr The number of bytes to add.
2134	*/
2135	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2136	{
2137	iemRegAddToRip(pVCpu, cbInstr);
2138	return iemRegFinishClearingRF(pVCpu);
2139	}
2140
2141
2142	/**
2143	* Updates the RIP to point to the next instruction and clears EFLAGS.RF
2144	* and CPUMCTX_INHIBIT_SHADOW.
2145	*
2146	* Only called from 64-bit code.
2147	*
2148	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2149	* @param cbInstr The number of bytes to add.
2150	*/
2151	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2152	{
2153	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2154	return iemRegFinishClearingRF(pVCpu);
2155	}
2156
2157
2158	/**
2159	* Updates the EIP to point to the next instruction and clears EFLAGS.RF and
2160	* CPUMCTX_INHIBIT_SHADOW.
2161	*
2162	* This is never from 64-bit code.
2163	*
2164	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2165	* @param cbInstr The number of bytes to add.
2166	*/
2167	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2168	{
2169	pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2170	return iemRegFinishClearingRF(pVCpu);
2171	}
2172
2173
2174	/**
2175	* Updates the IP to point to the next instruction and clears EFLAGS.RF and
2176	* CPUMCTX_INHIBIT_SHADOW.
2177	*
2178	* This is only ever used from 16-bit code on a pre-386 CPU.
2179	*
2180	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2181	* @param cbInstr The number of bytes to add.
2182	*/
2183	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2184	{
2185	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2186	return iemRegFinishClearingRF(pVCpu);
2187	}
2188
2189
2190	/**
2191	* Tail method for a finish function that does't clear flags or raise \#DB.
2192	*
2193	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2194	*/
2195	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishNoFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
2196	{
2197	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2198	Assert(!( pVCpu->cpum.GstCtx.eflags.uBoth
2199	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) );
2200	RT_NOREF(pVCpu);
2201	return VINF_SUCCESS;
2202	}
2203
2204
2205	/**
2206	* Updates the RIP to point to the next instruction, but does not need to clear
2207	* EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.
2208	*
2209	* Only called from 64-bit code.
2210	*
2211	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2212	* @param cbInstr The number of bytes to add.
2213	*/
2214	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2215	{
2216	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2217	return iemRegFinishNoFlags(pVCpu);
2218	}
2219
2220
2221	/**
2222	* Updates the EIP to point to the next instruction, but does not need to clear
2223	* EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.
2224	*
2225	* This is never from 64-bit code.
2226	*
2227	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2228	* @param cbInstr The number of bytes to add.
2229	*/
2230	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2231	{
2232	pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2233	return iemRegFinishNoFlags(pVCpu);
2234	}
2235
2236
2237	/**
2238	* Updates the IP to point to the next instruction, but does not need to clear
2239	* EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.
2240	*
2241	* This is only ever used from 16-bit code on a pre-386 CPU.
2242	*
2243	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2244	* @param cbInstr The number of bytes to add.
2245	*/
2246	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2247	{
2248	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2249	return iemRegFinishNoFlags(pVCpu);
2250	}
2251
2252
2253	/**
2254	* Adds a 8-bit signed jump offset to RIP from 64-bit code.
2255	*
2256	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2257	* segment limit.
2258	*
2259	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2260	* @param cbInstr Instruction size.
2261	* @param offNextInstr The offset of the next instruction.
2262	* @param enmEffOpSize Effective operand size.
2263	*/
2264	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2265	IEMMODE enmEffOpSize) RT_NOEXCEPT
2266	{
2267	Assert(IEM_IS_64BIT_CODE(pVCpu));
2268	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2269
2270	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2271	if (enmEffOpSize == IEMMODE_16BIT)
2272	uNewRip &= UINT16_MAX;
2273
2274	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2275	pVCpu->cpum.GstCtx.rip = uNewRip;
2276	else
2277	return iemRaiseGeneralProtectionFault0(pVCpu);
2278
2279	#ifndef IEM_WITH_CODE_TLB
2280	iemOpcodeFlushLight(pVCpu, cbInstr);
2281	#endif
2282
2283	/*
2284	* Clear RF and finish the instruction (maybe raise #DB).
2285	*/
2286	return iemRegFinishClearingRF(pVCpu);
2287	}
2288
2289
2290	/**
2291	* Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2292	* code (never 64-bit).
2293	*
2294	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2295	* segment limit.
2296	*
2297	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2298	* @param cbInstr Instruction size.
2299	* @param offNextInstr The offset of the next instruction.
2300	* @param enmEffOpSize Effective operand size.
2301	*/
2302	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2303	IEMMODE enmEffOpSize) RT_NOEXCEPT
2304	{
2305	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2306	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2307
2308	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2309	if (enmEffOpSize == IEMMODE_16BIT)
2310	uNewEip &= UINT16_MAX;
2311	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2312	pVCpu->cpum.GstCtx.rip = uNewEip;
2313	else
2314	return iemRaiseGeneralProtectionFault0(pVCpu);
2315
2316	#ifndef IEM_WITH_CODE_TLB
2317	iemOpcodeFlushLight(pVCpu, cbInstr);
2318	#endif
2319
2320	/*
2321	* Clear RF and finish the instruction (maybe raise #DB).
2322	*/
2323	return iemRegFinishClearingRF(pVCpu);
2324	}
2325
2326
2327	/**
2328	* Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.
2329	*
2330	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2331	* segment limit.
2332	*
2333	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2334	* @param cbInstr Instruction size.
2335	* @param offNextInstr The offset of the next instruction.
2336	*/
2337	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2338	int8_t offNextInstr) RT_NOEXCEPT
2339	{
2340	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2341
2342	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2343	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2344	pVCpu->cpum.GstCtx.rip = uNewIp;
2345	else
2346	return iemRaiseGeneralProtectionFault0(pVCpu);
2347
2348	#ifndef IEM_WITH_CODE_TLB
2349	iemOpcodeFlushLight(pVCpu, cbInstr);
2350	#endif
2351
2352	/*
2353	* Clear RF and finish the instruction (maybe raise #DB).
2354	*/
2355	return iemRegFinishClearingRF(pVCpu);
2356	}
2357
2358
2359	/**
2360	* Adds a 8-bit signed jump offset to RIP from 64-bit code, no checking or
2361	* clearing of flags.
2362	*
2363	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2364	* segment limit.
2365	*
2366	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2367	* @param cbInstr Instruction size.
2368	* @param offNextInstr The offset of the next instruction.
2369	* @param enmEffOpSize Effective operand size.
2370	*/
2371	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2372	IEMMODE enmEffOpSize) RT_NOEXCEPT
2373	{
2374	Assert(IEM_IS_64BIT_CODE(pVCpu));
2375	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2376
2377	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2378	if (enmEffOpSize == IEMMODE_16BIT)
2379	uNewRip &= UINT16_MAX;
2380
2381	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2382	pVCpu->cpum.GstCtx.rip = uNewRip;
2383	else
2384	return iemRaiseGeneralProtectionFault0(pVCpu);
2385
2386	#ifndef IEM_WITH_CODE_TLB
2387	iemOpcodeFlushLight(pVCpu, cbInstr);
2388	#endif
2389	return iemRegFinishNoFlags(pVCpu);
2390	}
2391
2392
2393	/**
2394	* Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2395	* code (never 64-bit), no checking or clearing of flags.
2396	*
2397	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2398	* segment limit.
2399	*
2400	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2401	* @param cbInstr Instruction size.
2402	* @param offNextInstr The offset of the next instruction.
2403	* @param enmEffOpSize Effective operand size.
2404	*/
2405	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2406	IEMMODE enmEffOpSize) RT_NOEXCEPT
2407	{
2408	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2409	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2410
2411	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2412	if (enmEffOpSize == IEMMODE_16BIT)
2413	uNewEip &= UINT16_MAX;
2414	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2415	pVCpu->cpum.GstCtx.rip = uNewEip;
2416	else
2417	return iemRaiseGeneralProtectionFault0(pVCpu);
2418
2419	#ifndef IEM_WITH_CODE_TLB
2420	iemOpcodeFlushLight(pVCpu, cbInstr);
2421	#endif
2422	return iemRegFinishNoFlags(pVCpu);
2423	}
2424
2425
2426	/**
2427	* Adds a 8-bit signed jump offset to IP, on a pre-386 CPU, no checking or
2428	* clearing of flags.
2429	*
2430	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2431	* segment limit.
2432	*
2433	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2434	* @param cbInstr Instruction size.
2435	* @param offNextInstr The offset of the next instruction.
2436	*/
2437	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2438	int8_t offNextInstr) RT_NOEXCEPT
2439	{
2440	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2441
2442	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2443	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2444	pVCpu->cpum.GstCtx.rip = uNewIp;
2445	else
2446	return iemRaiseGeneralProtectionFault0(pVCpu);
2447
2448	#ifndef IEM_WITH_CODE_TLB
2449	iemOpcodeFlushLight(pVCpu, cbInstr);
2450	#endif
2451	return iemRegFinishNoFlags(pVCpu);
2452	}
2453
2454
2455	/**
2456	* Adds a 16-bit signed jump offset to RIP from 64-bit code.
2457	*
2458	* @returns Strict VBox status code.
2459	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2460	* @param cbInstr Instruction size.
2461	* @param offNextInstr The offset of the next instruction.
2462	*/
2463	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2464	int16_t offNextInstr) RT_NOEXCEPT
2465	{
2466	Assert(IEM_IS_64BIT_CODE(pVCpu));
2467
2468	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2469
2470	#ifndef IEM_WITH_CODE_TLB
2471	iemOpcodeFlushLight(pVCpu, cbInstr);
2472	#endif
2473
2474	/*
2475	* Clear RF and finish the instruction (maybe raise #DB).
2476	*/
2477	return iemRegFinishClearingRF(pVCpu);
2478	}
2479
2480
2481	/**
2482	* Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.
2483	*
2484	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2485	* segment limit.
2486	*
2487	* @returns Strict VBox status code.
2488	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2489	* @param cbInstr Instruction size.
2490	* @param offNextInstr The offset of the next instruction.
2491	*
2492	* @note This is also used by 16-bit code in pre-386 mode, as the code is
2493	* identical.
2494	*/
2495	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2496	int16_t offNextInstr) RT_NOEXCEPT
2497	{
2498	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2499
2500	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2501	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2502	pVCpu->cpum.GstCtx.rip = uNewIp;
2503	else
2504	return iemRaiseGeneralProtectionFault0(pVCpu);
2505
2506	#ifndef IEM_WITH_CODE_TLB
2507	iemOpcodeFlushLight(pVCpu, cbInstr);
2508	#endif
2509
2510	/*
2511	* Clear RF and finish the instruction (maybe raise #DB).
2512	*/
2513	return iemRegFinishClearingRF(pVCpu);
2514	}
2515
2516
2517	/**
2518	* Adds a 16-bit signed jump offset to RIP from 64-bit code, no checking or
2519	* clearing of flags.
2520	*
2521	* @returns Strict VBox status code.
2522	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2523	* @param cbInstr Instruction size.
2524	* @param offNextInstr The offset of the next instruction.
2525	*/
2526	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2527	int16_t offNextInstr) RT_NOEXCEPT
2528	{
2529	Assert(IEM_IS_64BIT_CODE(pVCpu));
2530
2531	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2532
2533	#ifndef IEM_WITH_CODE_TLB
2534	iemOpcodeFlushLight(pVCpu, cbInstr);
2535	#endif
2536	return iemRegFinishNoFlags(pVCpu);
2537	}
2538
2539
2540	/**
2541	* Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code,
2542	* no checking or clearing of flags.
2543	*
2544	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2545	* segment limit.
2546	*
2547	* @returns Strict VBox status code.
2548	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2549	* @param cbInstr Instruction size.
2550	* @param offNextInstr The offset of the next instruction.
2551	*
2552	* @note This is also used by 16-bit code in pre-386 mode, as the code is
2553	* identical.
2554	*/
2555	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2556	int16_t offNextInstr) RT_NOEXCEPT
2557	{
2558	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2559
2560	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2561	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2562	pVCpu->cpum.GstCtx.rip = uNewIp;
2563	else
2564	return iemRaiseGeneralProtectionFault0(pVCpu);
2565
2566	#ifndef IEM_WITH_CODE_TLB
2567	iemOpcodeFlushLight(pVCpu, cbInstr);
2568	#endif
2569	return iemRegFinishNoFlags(pVCpu);
2570	}
2571
2572
2573	/**
2574	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2575	*
2576	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2577	* segment limit.
2578	*
2579	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2580	* only alternative for relative jumps in 64-bit code and that is already
2581	* handled in the decoder stage.
2582	*
2583	* @returns Strict VBox status code.
2584	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2585	* @param cbInstr Instruction size.
2586	* @param offNextInstr The offset of the next instruction.
2587	*/
2588	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2589	int32_t offNextInstr) RT_NOEXCEPT
2590	{
2591	Assert(IEM_IS_64BIT_CODE(pVCpu));
2592
2593	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2594	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2595	pVCpu->cpum.GstCtx.rip = uNewRip;
2596	else
2597	return iemRaiseGeneralProtectionFault0(pVCpu);
2598
2599	#ifndef IEM_WITH_CODE_TLB
2600	iemOpcodeFlushLight(pVCpu, cbInstr);
2601	#endif
2602
2603	/*
2604	* Clear RF and finish the instruction (maybe raise #DB).
2605	*/
2606	return iemRegFinishClearingRF(pVCpu);
2607	}
2608
2609
2610	/**
2611	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2612	*
2613	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2614	* segment limit.
2615	*
2616	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2617	* only alternative for relative jumps in 32-bit code and that is already
2618	* handled in the decoder stage.
2619	*
2620	* @returns Strict VBox status code.
2621	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2622	* @param cbInstr Instruction size.
2623	* @param offNextInstr The offset of the next instruction.
2624	*/
2625	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2626	int32_t offNextInstr) RT_NOEXCEPT
2627	{
2628	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2629	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2630
2631	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2632	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2633	pVCpu->cpum.GstCtx.rip = uNewEip;
2634	else
2635	return iemRaiseGeneralProtectionFault0(pVCpu);
2636
2637	#ifndef IEM_WITH_CODE_TLB
2638	iemOpcodeFlushLight(pVCpu, cbInstr);
2639	#endif
2640
2641	/*
2642	* Clear RF and finish the instruction (maybe raise #DB).
2643	*/
2644	return iemRegFinishClearingRF(pVCpu);
2645	}
2646
2647
2648	/**
2649	* Adds a 32-bit signed jump offset to RIP from 64-bit code, no checking or
2650	* clearing of flags.
2651	*
2652	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2653	* segment limit.
2654	*
2655	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2656	* only alternative for relative jumps in 64-bit code and that is already
2657	* handled in the decoder stage.
2658	*
2659	* @returns Strict VBox status code.
2660	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2661	* @param cbInstr Instruction size.
2662	* @param offNextInstr The offset of the next instruction.
2663	*/
2664	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2665	int32_t offNextInstr) RT_NOEXCEPT
2666	{
2667	Assert(IEM_IS_64BIT_CODE(pVCpu));
2668
2669	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2670	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2671	pVCpu->cpum.GstCtx.rip = uNewRip;
2672	else
2673	return iemRaiseGeneralProtectionFault0(pVCpu);
2674
2675	#ifndef IEM_WITH_CODE_TLB
2676	iemOpcodeFlushLight(pVCpu, cbInstr);
2677	#endif
2678	return iemRegFinishNoFlags(pVCpu);
2679	}
2680
2681
2682	/**
2683	* Adds a 32-bit signed jump offset to RIP from 64-bit code, no checking or
2684	* clearing of flags.
2685	*
2686	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2687	* segment limit.
2688	*
2689	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2690	* only alternative for relative jumps in 32-bit code and that is already
2691	* handled in the decoder stage.
2692	*
2693	* @returns Strict VBox status code.
2694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2695	* @param cbInstr Instruction size.
2696	* @param offNextInstr The offset of the next instruction.
2697	*/
2698	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2699	int32_t offNextInstr) RT_NOEXCEPT
2700	{
2701	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2702	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2703
2704	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2705	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2706	pVCpu->cpum.GstCtx.rip = uNewEip;
2707	else
2708	return iemRaiseGeneralProtectionFault0(pVCpu);
2709
2710	#ifndef IEM_WITH_CODE_TLB
2711	iemOpcodeFlushLight(pVCpu, cbInstr);
2712	#endif
2713	return iemRegFinishNoFlags(pVCpu);
2714	}
2715
2716
2717	/**
2718	* Extended version of iemFinishInstructionWithFlagsSet that goes with
2719	* iemRegAddToRipAndFinishingClearingRfEx.
2720	*
2721	* See iemFinishInstructionWithFlagsSet() for details.
2722	*/
2723	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2724	{
2725	/*
2726	* Raise a #DB.
2727	*/
2728	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2729	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2730	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
2731	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2732	/** @todo Do we set all pending \#DB events, or just one? */
2733	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
2734	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2735	pVCpu->cpum.GstCtx.rflags.uBoth));
2736	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
2737	return iemRaiseDebugException(pVCpu);
2738	}
2739
2740
2741	/**
2742	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
2743	* others potentially updating EFLAGS.TF.
2744	*
2745	* The single step event must be generated using the TF value at the start of
2746	* the instruction, not the new value set by it.
2747	*
2748	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2749	* @param cbInstr The number of bytes to add.
2750	* @param fEflOld The EFLAGS at the start of the instruction
2751	* execution.
2752	*/
2753	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
2754	{
2755	iemRegAddToRip(pVCpu, cbInstr);
2756	if (!(fEflOld & X86_EFL_TF))
2757	return iemRegFinishClearingRF(pVCpu);
2758	return iemFinishInstructionWithTfSet(pVCpu);
2759	}
2760
2761
2762	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2763	/**
2764	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
2765	*
2766	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2767	*/
2768	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2769	{
2770	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
2771	}
2772	#endif
2773
2774
2775
2776	/**
2777	* Performs a near jump to the specified address, no checking or clearing of
2778	* flags
2779	*
2780	* May raise a \#GP(0) if the new IP outside the code segment limit.
2781	*
2782	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2783	* @param uNewIp The new IP value.
2784	*/
2785	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishNoFlags(PVMCPUCC pVCpu, uint16_t uNewIp) RT_NOEXCEPT
2786	{
2787	if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
2788	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))
2789	pVCpu->cpum.GstCtx.rip = uNewIp;
2790	else
2791	return iemRaiseGeneralProtectionFault0(pVCpu);
2792	#ifndef IEM_WITH_CODE_TLB
2793	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
2794	#endif
2795	return iemRegFinishNoFlags(pVCpu);
2796	}
2797
2798
2799	/**
2800	* Performs a near jump to the specified address, no checking or clearing of
2801	* flags
2802	*
2803	* May raise a \#GP(0) if the new RIP is outside the code segment limit.
2804	*
2805	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2806	* @param uNewEip The new EIP value.
2807	*/
2808	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishNoFlags(PVMCPUCC pVCpu, uint32_t uNewEip) RT_NOEXCEPT
2809	{
2810	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2811	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2812	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2813	pVCpu->cpum.GstCtx.rip = uNewEip;
2814	else
2815	return iemRaiseGeneralProtectionFault0(pVCpu);
2816	#ifndef IEM_WITH_CODE_TLB
2817	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
2818	#endif
2819	return iemRegFinishNoFlags(pVCpu);
2820	}
2821
2822
2823	/**
2824	* Performs a near jump to the specified address, no checking or clearing of
2825	* flags.
2826	*
2827	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2828	* segment limit.
2829	*
2830	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2831	* @param uNewRip The new RIP value.
2832	*/
2833	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishNoFlags(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT
2834	{
2835	Assert(IEM_IS_64BIT_CODE(pVCpu));
2836	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2837	pVCpu->cpum.GstCtx.rip = uNewRip;
2838	else
2839	return iemRaiseGeneralProtectionFault0(pVCpu);
2840	#ifndef IEM_WITH_CODE_TLB
2841	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
2842	#endif
2843	return iemRegFinishNoFlags(pVCpu);
2844	}
2845
2846
2847	/**
2848	* Performs a near jump to the specified address.
2849	*
2850	* May raise a \#GP(0) if the new IP outside the code segment limit.
2851	*
2852	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2853	* @param uNewIp The new IP value.
2854	*/
2855	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishClearingRF(PVMCPUCC pVCpu, uint16_t uNewIp) RT_NOEXCEPT
2856	{
2857	if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
2858	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))
2859	pVCpu->cpum.GstCtx.rip = uNewIp;
2860	else
2861	return iemRaiseGeneralProtectionFault0(pVCpu);
2862	#ifndef IEM_WITH_CODE_TLB
2863	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
2864	#endif
2865	return iemRegFinishClearingRF(pVCpu);
2866	}
2867
2868
2869	/**
2870	* Performs a near jump to the specified address.
2871	*
2872	* May raise a \#GP(0) if the new RIP is outside the code segment limit.
2873	*
2874	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2875	* @param uNewEip The new EIP value.
2876	*/
2877	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishClearingRF(PVMCPUCC pVCpu, uint32_t uNewEip) RT_NOEXCEPT
2878	{
2879	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2880	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2881	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2882	pVCpu->cpum.GstCtx.rip = uNewEip;
2883	else
2884	return iemRaiseGeneralProtectionFault0(pVCpu);
2885	#ifndef IEM_WITH_CODE_TLB
2886	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
2887	#endif
2888	return iemRegFinishClearingRF(pVCpu);
2889	}
2890
2891
2892	/**
2893	* Performs a near jump to the specified address.
2894	*
2895	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2896	* segment limit.
2897	*
2898	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2899	* @param uNewRip The new RIP value.
2900	*/
2901	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishClearingRF(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT
2902	{
2903	Assert(IEM_IS_64BIT_CODE(pVCpu));
2904	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2905	pVCpu->cpum.GstCtx.rip = uNewRip;
2906	else
2907	return iemRaiseGeneralProtectionFault0(pVCpu);
2908	#ifndef IEM_WITH_CODE_TLB
2909	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
2910	#endif
2911	return iemRegFinishClearingRF(pVCpu);
2912	}
2913
2914
2915
2916	/**
2917	* Adds to the stack pointer.
2918	*
2919	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2920	* @param cbToAdd The number of bytes to add (8-bit!).
2921	*/
2922	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2923	{
2924	if (IEM_IS_64BIT_CODE(pVCpu))
2925	pVCpu->cpum.GstCtx.rsp += cbToAdd;
2926	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2927	pVCpu->cpum.GstCtx.esp += cbToAdd;
2928	else
2929	pVCpu->cpum.GstCtx.sp += cbToAdd;
2930	}
2931
2932
2933	/**
2934	* Subtracts from the stack pointer.
2935	*
2936	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2937	* @param cbToSub The number of bytes to subtract (8-bit!).
2938	*/
2939	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2940	{
2941	if (IEM_IS_64BIT_CODE(pVCpu))
2942	pVCpu->cpum.GstCtx.rsp -= cbToSub;
2943	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2944	pVCpu->cpum.GstCtx.esp -= cbToSub;
2945	else
2946	pVCpu->cpum.GstCtx.sp -= cbToSub;
2947	}
2948
2949
2950	/**
2951	* Adds to the temporary stack pointer.
2952	*
2953	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2954	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2955	* @param cbToAdd The number of bytes to add (16-bit).
2956	*/
2957	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2958	{
2959	if (IEM_IS_64BIT_CODE(pVCpu))
2960	pTmpRsp->u += cbToAdd;
2961	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2962	pTmpRsp->DWords.dw0 += cbToAdd;
2963	else
2964	pTmpRsp->Words.w0 += cbToAdd;
2965	}
2966
2967
2968	/**
2969	* Subtracts from the temporary stack pointer.
2970	*
2971	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2972	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2973	* @param cbToSub The number of bytes to subtract.
2974	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
2975	* expecting that.
2976	*/
2977	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2978	{
2979	if (IEM_IS_64BIT_CODE(pVCpu))
2980	pTmpRsp->u -= cbToSub;
2981	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2982	pTmpRsp->DWords.dw0 -= cbToSub;
2983	else
2984	pTmpRsp->Words.w0 -= cbToSub;
2985	}
2986
2987
2988	/**
2989	* Calculates the effective stack address for a push of the specified size as
2990	* well as the new RSP value (upper bits may be masked).
2991	*
2992	* @returns Effective stack addressf for the push.
2993	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2994	* @param cbItem The size of the stack item to pop.
2995	* @param puNewRsp Where to return the new RSP value.
2996	*/
2997	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2998	{
2999	RTUINT64U uTmpRsp;
3000	RTGCPTR GCPtrTop;
3001	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
3002
3003	if (IEM_IS_64BIT_CODE(pVCpu))
3004	GCPtrTop = uTmpRsp.u -= cbItem;
3005	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3006	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
3007	else
3008	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
3009	*puNewRsp = uTmpRsp.u;
3010	return GCPtrTop;
3011	}
3012
3013
3014	/**
3015	* Gets the current stack pointer and calculates the value after a pop of the
3016	* specified size.
3017	*
3018	* @returns Current stack pointer.
3019	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3020	* @param cbItem The size of the stack item to pop.
3021	* @param puNewRsp Where to return the new RSP value.
3022	*/
3023	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
3024	{
3025	RTUINT64U uTmpRsp;
3026	RTGCPTR GCPtrTop;
3027	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
3028
3029	if (IEM_IS_64BIT_CODE(pVCpu))
3030	{
3031	GCPtrTop = uTmpRsp.u;
3032	uTmpRsp.u += cbItem;
3033	}
3034	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3035	{
3036	GCPtrTop = uTmpRsp.DWords.dw0;
3037	uTmpRsp.DWords.dw0 += cbItem;
3038	}
3039	else
3040	{
3041	GCPtrTop = uTmpRsp.Words.w0;
3042	uTmpRsp.Words.w0 += cbItem;
3043	}
3044	*puNewRsp = uTmpRsp.u;
3045	return GCPtrTop;
3046	}
3047
3048
3049	/**
3050	* Calculates the effective stack address for a push of the specified size as
3051	* well as the new temporary RSP value (upper bits may be masked).
3052	*
3053	* @returns Effective stack addressf for the push.
3054	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3055	* @param pTmpRsp The temporary stack pointer. This is updated.
3056	* @param cbItem The size of the stack item to pop.
3057	*/
3058	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
3059	{
3060	RTGCPTR GCPtrTop;
3061
3062	if (IEM_IS_64BIT_CODE(pVCpu))
3063	GCPtrTop = pTmpRsp->u -= cbItem;
3064	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3065	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
3066	else
3067	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
3068	return GCPtrTop;
3069	}
3070
3071
3072	/**
3073	* Gets the effective stack address for a pop of the specified size and
3074	* calculates and updates the temporary RSP.
3075	*
3076	* @returns Current stack pointer.
3077	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3078	* @param pTmpRsp The temporary stack pointer. This is updated.
3079	* @param cbItem The size of the stack item to pop.
3080	*/
3081	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
3082	{
3083	RTGCPTR GCPtrTop;
3084	if (IEM_IS_64BIT_CODE(pVCpu))
3085	{
3086	GCPtrTop = pTmpRsp->u;
3087	pTmpRsp->u += cbItem;
3088	}
3089	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3090	{
3091	GCPtrTop = pTmpRsp->DWords.dw0;
3092	pTmpRsp->DWords.dw0 += cbItem;
3093	}
3094	else
3095	{
3096	GCPtrTop = pTmpRsp->Words.w0;
3097	pTmpRsp->Words.w0 += cbItem;
3098	}
3099	return GCPtrTop;
3100	}
3101
3102	/** @} */
3103
3104
3105	/** @name FPU access and helpers.
3106	*
3107	* @{
3108	*/
3109
3110
3111	/**
3112	* Hook for preparing to use the host FPU.
3113	*
3114	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3115	*
3116	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3117	*/
3118	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
3119	{
3120	#ifdef IN_RING3
3121	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
3122	#else
3123	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
3124	#endif
3125	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3126	}
3127
3128
3129	/**
3130	* Hook for preparing to use the host FPU for SSE.
3131	*
3132	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3133	*
3134	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3135	*/
3136	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
3137	{
3138	iemFpuPrepareUsage(pVCpu);
3139	}
3140
3141
3142	/**
3143	* Hook for preparing to use the host FPU for AVX.
3144	*
3145	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3146	*
3147	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3148	*/
3149	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
3150	{
3151	iemFpuPrepareUsage(pVCpu);
3152	}
3153
3154
3155	/**
3156	* Hook for actualizing the guest FPU state before the interpreter reads it.
3157	*
3158	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3159	*
3160	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3161	*/
3162	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
3163	{
3164	#ifdef IN_RING3
3165	NOREF(pVCpu);
3166	#else
3167	CPUMRZFpuStateActualizeForRead(pVCpu);
3168	#endif
3169	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3170	}
3171
3172
3173	/**
3174	* Hook for actualizing the guest FPU state before the interpreter changes it.
3175	*
3176	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3177	*
3178	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3179	*/
3180	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
3181	{
3182	#ifdef IN_RING3
3183	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
3184	#else
3185	CPUMRZFpuStateActualizeForChange(pVCpu);
3186	#endif
3187	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3188	}
3189
3190
3191	/**
3192	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
3193	* only.
3194	*
3195	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3196	*
3197	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3198	*/
3199	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
3200	{
3201	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
3202	NOREF(pVCpu);
3203	#else
3204	CPUMRZFpuStateActualizeSseForRead(pVCpu);
3205	#endif
3206	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3207	}
3208
3209
3210	/**
3211	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
3212	* read+write.
3213	*
3214	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3215	*
3216	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3217	*/
3218	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
3219	{
3220	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
3221	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
3222	#else
3223	CPUMRZFpuStateActualizeForChange(pVCpu);
3224	#endif
3225	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3226
3227	/* Make sure any changes are loaded the next time around. */
3228	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
3229	}
3230
3231
3232	/**
3233	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
3234	* only.
3235	*
3236	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3237	*
3238	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3239	*/
3240	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
3241	{
3242	#ifdef IN_RING3
3243	NOREF(pVCpu);
3244	#else
3245	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
3246	#endif
3247	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3248	}
3249
3250
3251	/**
3252	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
3253	* read+write.
3254	*
3255	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3256	*
3257	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3258	*/
3259	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
3260	{
3261	#ifdef IN_RING3
3262	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
3263	#else
3264	CPUMRZFpuStateActualizeForChange(pVCpu);
3265	#endif
3266	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3267
3268	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
3269	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
3270	}
3271
3272
3273	/**
3274	* Stores a QNaN value into a FPU register.
3275	*
3276	* @param pReg Pointer to the register.
3277	*/
3278	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
3279	{
3280	pReg->au32[0] = UINT32_C(0x00000000);
3281	pReg->au32[1] = UINT32_C(0xc0000000);
3282	pReg->au16[4] = UINT16_C(0xffff);
3283	}
3284
3285
3286	/**
3287	* Updates the FOP, FPU.CS and FPUIP registers, extended version.
3288	*
3289	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3290	* @param pFpuCtx The FPU context.
3291	* @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
3292	*/
3293	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
3294	{
3295	Assert(uFpuOpcode != UINT16_MAX);
3296	pFpuCtx->FOP = uFpuOpcode;
3297	/** @todo x87.CS and FPUIP needs to be kept seperately. */
3298	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
3299	{
3300	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
3301	* happens in real mode here based on the fnsave and fnstenv images. */
3302	pFpuCtx->CS = 0;
3303	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
3304	}
3305	else if (!IEM_IS_LONG_MODE(pVCpu))
3306	{
3307	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
3308	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
3309	}
3310	else
3311	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
3312	}
3313
3314
3315	/**
3316	* Marks the specified stack register as free (for FFREE).
3317	*
3318	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3319	* @param iStReg The register to free.
3320	*/
3321	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
3322	{
3323	Assert(iStReg < 8);
3324	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3325	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
3326	pFpuCtx->FTW &= ~RT_BIT(iReg);
3327	}
3328
3329
3330	/**
3331	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
3332	*
3333	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3334	*/
3335	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
3336	{
3337	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3338	uint16_t uFsw = pFpuCtx->FSW;
3339	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
3340	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
3341	uFsw &= ~X86_FSW_TOP_MASK;
3342	uFsw \|= uTop;
3343	pFpuCtx->FSW = uFsw;
3344	}
3345
3346
3347	/**
3348	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
3349	*
3350	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3351	*/
3352	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
3353	{
3354	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3355	uint16_t uFsw = pFpuCtx->FSW;
3356	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
3357	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
3358	uFsw &= ~X86_FSW_TOP_MASK;
3359	uFsw \|= uTop;
3360	pFpuCtx->FSW = uFsw;
3361	}
3362
3363
3364
3365
3366	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
3367	{
3368	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3369	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
3370	if (pFpuCtx->FTW & RT_BIT(iReg))
3371	return VINF_SUCCESS;
3372	return VERR_NOT_FOUND;
3373	}
3374
3375
3376	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
3377	{
3378	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3379	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
3380	if (pFpuCtx->FTW & RT_BIT(iReg))
3381	{
3382	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
3383	return VINF_SUCCESS;
3384	}
3385	return VERR_NOT_FOUND;
3386	}
3387
3388
3389	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
3390	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
3391	{
3392	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3393	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3394	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
3395	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
3396	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
3397	{
3398	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
3399	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
3400	return VINF_SUCCESS;
3401	}
3402	return VERR_NOT_FOUND;
3403	}
3404
3405
3406	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
3407	{
3408	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3409	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3410	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
3411	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
3412	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
3413	{
3414	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
3415	return VINF_SUCCESS;
3416	}
3417	return VERR_NOT_FOUND;
3418	}
3419
3420
3421	/**
3422	* Rotates the stack registers when setting new TOS.
3423	*
3424	* @param pFpuCtx The FPU context.
3425	* @param iNewTop New TOS value.
3426	* @remarks We only do this to speed up fxsave/fxrstor which
3427	* arrange the FP registers in stack order.
3428	* MUST be done before writing the new TOS (FSW).
3429	*/
3430	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
3431	{
3432	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3433	RTFLOAT80U ar80Temp[8];
3434
3435	if (iOldTop == iNewTop)
3436	return;
3437
3438	/* Unscrew the stack and get it into 'native' order. */
3439	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
3440	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
3441	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
3442	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
3443	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
3444	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
3445	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
3446	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
3447
3448	/* Now rotate the stack to the new position. */
3449	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
3450	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
3451	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
3452	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
3453	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
3454	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
3455	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
3456	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
3457	}
3458
3459
3460	/**
3461	* Updates the FPU exception status after FCW is changed.
3462	*
3463	* @param pFpuCtx The FPU context.
3464	*/
3465	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
3466	{
3467	uint16_t u16Fsw = pFpuCtx->FSW;
3468	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
3469	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
3470	else
3471	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
3472	pFpuCtx->FSW = u16Fsw;
3473	}
3474
3475
3476	/**
3477	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
3478	*
3479	* @returns The full FTW.
3480	* @param pFpuCtx The FPU context.
3481	*/
3482	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
3483	{
3484	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
3485	uint16_t u16Ftw = 0;
3486	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3487	for (unsigned iSt = 0; iSt < 8; iSt++)
3488	{
3489	unsigned const iReg = (iSt + iTop) & 7;
3490	if (!(u8Ftw & RT_BIT(iReg)))
3491	u16Ftw \|= 3 << (iReg * 2); /* empty */
3492	else
3493	{
3494	uint16_t uTag;
3495	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
3496	if (pr80Reg->s.uExponent == 0x7fff)
3497	uTag = 2; /* Exponent is all 1's => Special. */
3498	else if (pr80Reg->s.uExponent == 0x0000)
3499	{
3500	if (pr80Reg->s.uMantissa == 0x0000)
3501	uTag = 1; /* All bits are zero => Zero. */
3502	else
3503	uTag = 2; /* Must be special. */
3504	}
3505	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
3506	uTag = 0; /* Valid. */
3507	else
3508	uTag = 2; /* Must be special. */
3509
3510	u16Ftw \|= uTag << (iReg * 2);
3511	}
3512	}
3513
3514	return u16Ftw;
3515	}
3516
3517
3518	/**
3519	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
3520	*
3521	* @returns The compressed FTW.
3522	* @param u16FullFtw The full FTW to convert.
3523	*/
3524	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
3525	{
3526	uint8_t u8Ftw = 0;
3527	for (unsigned i = 0; i < 8; i++)
3528	{
3529	if ((u16FullFtw & 3) != 3 /empty/)
3530	u8Ftw \|= RT_BIT(i);
3531	u16FullFtw >>= 2;
3532	}
3533
3534	return u8Ftw;
3535	}
3536
3537	/** @} */
3538
3539
3540	/** @name Memory access.
3541	*
3542	* @{
3543	*/
3544
3545
3546	/**
3547	* Checks whether alignment checks are enabled or not.
3548	*
3549	* @returns true if enabled, false if not.
3550	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3551	*/
3552	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
3553	{
3554	AssertCompile(X86_CR0_AM == X86_EFL_AC);
3555	return IEM_GET_CPL(pVCpu) == 3
3556	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
3557	}
3558
3559	/**
3560	* Checks if the given segment can be written to, raise the appropriate
3561	* exception if not.
3562	*
3563	* @returns VBox strict status code.
3564	*
3565	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3566	* @param pHid Pointer to the hidden register.
3567	* @param iSegReg The register number.
3568	* @param pu64BaseAddr Where to return the base address to use for the
3569	* segment. (In 64-bit code it may differ from the
3570	* base in the hidden segment.)
3571	*/
3572	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3573	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3574	{
3575	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3576
3577	if (IEM_IS_64BIT_CODE(pVCpu))
3578	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3579	else
3580	{
3581	if (!pHid->Attr.n.u1Present)
3582	{
3583	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3584	AssertRelease(uSel == 0);
3585	LogEx(LOG_GROUP_IEM,("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3586	return iemRaiseGeneralProtectionFault0(pVCpu);
3587	}
3588
3589	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
3590	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3591	&& !IEM_IS_64BIT_CODE(pVCpu) )
3592	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3593	*pu64BaseAddr = pHid->u64Base;
3594	}
3595	return VINF_SUCCESS;
3596	}
3597
3598
3599	/**
3600	* Checks if the given segment can be read from, raise the appropriate
3601	* exception if not.
3602	*
3603	* @returns VBox strict status code.
3604	*
3605	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3606	* @param pHid Pointer to the hidden register.
3607	* @param iSegReg The register number.
3608	* @param pu64BaseAddr Where to return the base address to use for the
3609	* segment. (In 64-bit code it may differ from the
3610	* base in the hidden segment.)
3611	*/
3612	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3613	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3614	{
3615	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3616
3617	if (IEM_IS_64BIT_CODE(pVCpu))
3618	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3619	else
3620	{
3621	if (!pHid->Attr.n.u1Present)
3622	{
3623	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3624	AssertRelease(uSel == 0);
3625	LogEx(LOG_GROUP_IEM,("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3626	return iemRaiseGeneralProtectionFault0(pVCpu);
3627	}
3628
3629	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3630	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3631	*pu64BaseAddr = pHid->u64Base;
3632	}
3633	return VINF_SUCCESS;
3634	}
3635
3636
3637	/**
3638	* Maps a physical page.
3639	*
3640	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
3641	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3642	* @param GCPhysMem The physical address.
3643	* @param fAccess The intended access.
3644	* @param ppvMem Where to return the mapping address.
3645	* @param pLock The PGM lock.
3646	*/
3647	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3648	void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3649	{
3650	#ifdef IEM_LOG_MEMORY_WRITES
3651	if (fAccess & IEM_ACCESS_TYPE_WRITE)
3652	return VERR_PGM_PHYS_TLB_CATCH_ALL;
3653	#endif
3654
3655	/** @todo This API may require some improving later. A private deal with PGM
3656	* regarding locking and unlocking needs to be struct. A couple of TLBs
3657	* living in PGM, but with publicly accessible inlined access methods
3658	* could perhaps be an even better solution. */
3659	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
3660	GCPhysMem,
3661	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
3662	RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),
3663	ppvMem,
3664	pLock);
3665	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
3666	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
3667
3668	return rc;
3669	}
3670
3671
3672	/**
3673	* Unmap a page previously mapped by iemMemPageMap.
3674	*
3675	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3676	* @param GCPhysMem The physical address.
3677	* @param fAccess The intended access.
3678	* @param pvMem What iemMemPageMap returned.
3679	* @param pLock The PGM lock.
3680	*/
3681	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3682	const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3683	{
3684	NOREF(pVCpu);
3685	NOREF(GCPhysMem);
3686	NOREF(fAccess);
3687	NOREF(pvMem);
3688	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
3689	}
3690
3691	#ifdef IEM_WITH_SETJMP
3692
3693	/** @todo slim this down */
3694	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
3695	size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3696	{
3697	Assert(cbMem >= 1);
3698	Assert(iSegReg < X86_SREG_COUNT);
3699
3700	/*
3701	* 64-bit mode is simpler.
3702	*/
3703	if (IEM_IS_64BIT_CODE(pVCpu))
3704	{
3705	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
3706	{
3707	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3708	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3709	GCPtrMem += pSel->u64Base;
3710	}
3711
3712	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3713	return GCPtrMem;
3714	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3715	}
3716	/*
3717	* 16-bit and 32-bit segmentation.
3718	*/
3719	else if (iSegReg != UINT8_MAX)
3720	{
3721	/** @todo Does this apply to segments with 4G-1 limit? */
3722	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3723	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
3724	{
3725	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3726	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3727	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3728	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
3729	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
3730	\| X86_SEL_TYPE_CODE))
3731	{
3732	case X86DESCATTR_P: /* readonly data, expand up */
3733	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
3734	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
3735	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
3736	/* expand up */
3737	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
3738	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3739	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
3740	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
3741	break;
3742
3743	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
3744	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
3745	/* expand down */
3746	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3747	&& ( pSel->Attr.n.u1DefBig
3748	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
3749	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3750	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
3751	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
3752	break;
3753
3754	default:
3755	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
3756	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3757	break;
3758	}
3759	}
3760	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
3761	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3762	}
3763	/*
3764	* 32-bit flat address.
3765	*/
3766	else
3767	return GCPtrMem;
3768	}
3769
3770
3771	/** @todo slim this down */
3772	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
3773	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3774	{
3775	Assert(cbMem >= 1);
3776	Assert(iSegReg < X86_SREG_COUNT);
3777
3778	/*
3779	* 64-bit mode is simpler.
3780	*/
3781	if (IEM_IS_64BIT_CODE(pVCpu))
3782	{
3783	if (iSegReg >= X86_SREG_FS)
3784	{
3785	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3786	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3787	GCPtrMem += pSel->u64Base;
3788	}
3789
3790	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3791	return GCPtrMem;
3792	}
3793	/*
3794	* 16-bit and 32-bit segmentation.
3795	*/
3796	else
3797	{
3798	Assert(GCPtrMem <= UINT32_MAX);
3799	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3800	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3801	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3802	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
3803	if ( fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE) /* data, expand up */
3804	/** @todo explore exactly how the CS stuff works in real mode. See also
3805	* http://www.rcollins.org/Productivity/DescriptorCache.html and
3806	* http://www.rcollins.org/ddj/Aug98/Aug98.html for some insight. */
3807	\|\| (iSegReg == X86_SREG_CS && IEM_IS_REAL_OR_V86_MODE(pVCpu)) ) /* Ignored for CS. / /* @todo testcase! */
3808	{
3809	/* expand up */
3810	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3811	if (RT_LIKELY( GCPtrLast32 <= pSel->u32Limit
3812	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3813	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3814	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3815	}
3816	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
3817	{
3818	/* expand down - the uppger boundary is defined by the B bit, not G. */
3819	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3820	if (RT_LIKELY( (uint32_t)GCPtrMem >= pSel->u32Limit
3821	&& (pSel->Attr.n.u1DefBig \|\| GCPtrLast32 <= UINT32_C(0xffff))
3822	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3823	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3824	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3825	}
3826	else
3827	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3828	}
3829	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3830	}
3831
3832	#endif /* IEM_WITH_SETJMP */
3833
3834	/**
3835	* Fakes a long mode stack selector for SS = 0.
3836	*
3837	* @param pDescSs Where to return the fake stack descriptor.
3838	* @param uDpl The DPL we want.
3839	*/
3840	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
3841	{
3842	pDescSs->Long.au64[0] = 0;
3843	pDescSs->Long.au64[1] = 0;
3844	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
3845	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
3846	pDescSs->Long.Gen.u2Dpl = uDpl;
3847	pDescSs->Long.Gen.u1Present = 1;
3848	pDescSs->Long.Gen.u1Long = 1;
3849	}
3850
3851
3852	/*
3853	* Unmap helpers.
3854	*/
3855
3856	#ifdef IEM_WITH_SETJMP
3857
3858	DECL_INLINE_THROW(void) iemMemCommitAndUnmapRwJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3859	{
3860	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3861	if (RT_LIKELY(bMapInfo == 0))
3862	return;
3863	# endif
3864	iemMemCommitAndUnmapRwSafeJmp(pVCpu, bMapInfo);
3865	}
3866
3867
3868	DECL_INLINE_THROW(void) iemMemCommitAndUnmapWoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3869	{
3870	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3871	if (RT_LIKELY(bMapInfo == 0))
3872	return;
3873	# endif
3874	iemMemCommitAndUnmapWoSafeJmp(pVCpu, bMapInfo);
3875	}
3876
3877
3878	DECL_INLINE_THROW(void) iemMemCommitAndUnmapRoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3879	{
3880	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3881	if (RT_LIKELY(bMapInfo == 0))
3882	return;
3883	# endif
3884	iemMemCommitAndUnmapRoSafeJmp(pVCpu, bMapInfo);
3885	}
3886
3887	DECLINLINE(void) iemMemRollbackAndUnmapWo(PVMCPUCC pVCpu, uint8_t bMapInfo) RT_NOEXCEPT
3888	{
3889	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3890	if (RT_LIKELY(bMapInfo == 0))
3891	return;
3892	# endif
3893	iemMemRollbackAndUnmapWoSafe(pVCpu, bMapInfo);
3894	}
3895
3896	#endif /* IEM_WITH_SETJMP */
3897
3898
3899	/*
3900	* Instantiate R/W inline templates.
3901	*/
3902
3903	/** @def TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
3904	* Used to check if an unaligned access is if within the page and won't
3905	* trigger an \#AC.
3906	*
3907	* This can be used to deal with misaligned accesses on platforms that are
3908	* senstive to such if desires.
3909	*/
3910	AssertCompile(X86_CR0_AM == X86_EFL_AC);
3911	AssertCompile(((3U + 1U) << 16) == X86_CR0_AM);
3912	#if 1
3913	# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) \
3914	( ((a_GCPtrEff) & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(a_TmplMemType) \
3915	&& !( (uint32_t)(a_pVCpu)->cpum.GstCtx.cr0 \
3916	& (a_pVCpu)->cpum.GstCtx.eflags.u \
3917	& ((IEM_GET_CPL((a_pVCpu)) + 1U) << 16) /* IEM_GET_CPL(a_pVCpu) == 3 ? X86_CR0_AM : 0 */ \
3918	& X86_CR0_AM) )
3919	#else
3920	# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 0
3921	#endif
3922
3923	#define TMPL_MEM_TYPE uint8_t
3924	#define TMPL_MEM_TYPE_ALIGN 0
3925	#define TMPL_MEM_TYPE_SIZE 1
3926	#define TMPL_MEM_FN_SUFF U8
3927	#define TMPL_MEM_FMT_TYPE "%#04x"
3928	#define TMPL_MEM_FMT_DESC "byte"
3929	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3930
3931	#define TMPL_MEM_WITH_STACK
3932
3933	#define TMPL_MEM_TYPE uint16_t
3934	#define TMPL_MEM_TYPE_ALIGN 1
3935	#define TMPL_MEM_TYPE_SIZE 2
3936	#define TMPL_MEM_FN_SUFF U16
3937	#define TMPL_MEM_FMT_TYPE "%#06x"
3938	#define TMPL_MEM_FMT_DESC "word"
3939	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3940
3941	#define TMPL_WITH_PUSH_SREG
3942	#define TMPL_MEM_TYPE uint32_t
3943	#define TMPL_MEM_TYPE_ALIGN 3
3944	#define TMPL_MEM_TYPE_SIZE 4
3945	#define TMPL_MEM_FN_SUFF U32
3946	#define TMPL_MEM_FMT_TYPE "%#010x"
3947	#define TMPL_MEM_FMT_DESC "dword"
3948	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3949	#undef TMPL_WITH_PUSH_SREG
3950
3951	#define TMPL_MEM_TYPE uint64_t
3952	#define TMPL_MEM_TYPE_ALIGN 7
3953	#define TMPL_MEM_TYPE_SIZE 8
3954	#define TMPL_MEM_FN_SUFF U64
3955	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3956	#define TMPL_MEM_FMT_DESC "qword"
3957	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3958
3959	#undef TMPL_MEM_WITH_STACK
3960
3961	#define TMPL_MEM_NO_STORE
3962	#define TMPL_MEM_NO_MAPPING
3963	#define TMPL_MEM_TYPE uint64_t
3964	#define TMPL_MEM_TYPE_ALIGN 15
3965	#define TMPL_MEM_TYPE_SIZE 8
3966	#define TMPL_MEM_FN_SUFF U64AlignedU128
3967	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3968	#define TMPL_MEM_FMT_DESC "qword"
3969	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3970
3971	#undef TMPL_MEM_NO_STORE
3972	#undef TMPL_MEM_NO_MAPPING
3973
3974	#define TMPL_MEM_TYPE RTFLOAT80U
3975	#define TMPL_MEM_TYPE_ALIGN 7
3976	#define TMPL_MEM_TYPE_SIZE 10
3977	#define TMPL_MEM_FN_SUFF R80
3978	#define TMPL_MEM_FMT_TYPE "%.10Rhxs"
3979	#define TMPL_MEM_FMT_DESC "tword"
3980	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3981
3982	#define TMPL_MEM_TYPE RTPBCD80U
3983	#define TMPL_MEM_TYPE_ALIGN 7 /** @todo RTPBCD80U alignment testcase */
3984	#define TMPL_MEM_TYPE_SIZE 10
3985	#define TMPL_MEM_FN_SUFF D80
3986	#define TMPL_MEM_FMT_TYPE "%.10Rhxs"
3987	#define TMPL_MEM_FMT_DESC "tword"
3988	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3989
3990	#define TMPL_MEM_TYPE RTUINT128U
3991	#define TMPL_MEM_TYPE_ALIGN 15
3992	#define TMPL_MEM_TYPE_SIZE 16
3993	#define TMPL_MEM_FN_SUFF U128
3994	#define TMPL_MEM_FMT_TYPE "%.16Rhxs"
3995	#define TMPL_MEM_FMT_DESC "dqword"
3996	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3997
3998	#undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
3999
4000	/** @} */
4001
4002
4003	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4004
4005	/**
4006	* Gets CR0 fixed-0 bits in VMX operation.
4007	*
4008	* We do this rather than fetching what we report to the guest (in
4009	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
4010	* values regardless of whether unrestricted-guest feature is available on the CPU.
4011	*
4012	* @returns CR0 fixed-0 bits.
4013	* @param pVCpu The cross context virtual CPU structure.
4014	* @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
4015	* must be returned. When @c false, the CR0 fixed-0
4016	* bits for VMX root mode is returned.
4017	*
4018	*/
4019	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
4020	{
4021	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
4022
4023	PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
4024	if ( fVmxNonRootMode
4025	&& (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
4026	return VMX_V_CR0_FIXED0_UX;
4027	return VMX_V_CR0_FIXED0;
4028	}
4029
4030
4031	# ifdef XAPIC_OFF_END /* Requires VBox/apic.h to be included before IEMInline.h. */
4032	/**
4033	* Sets virtual-APIC write emulation as pending.
4034	*
4035	* @param pVCpu The cross context virtual CPU structure.
4036	* @param offApic The offset in the virtual-APIC page that was written.
4037	*/
4038	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
4039	{
4040	Assert(offApic < XAPIC_OFF_END + 4);
4041
4042	/*
4043	* Record the currently updated APIC offset, as we need this later for figuring
4044	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
4045	* as for supplying the exit qualification when causing an APIC-write VM-exit.
4046	*/
4047	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
4048
4049	/*
4050	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
4051	* virtualization or APIC-write emulation).
4052	*/
4053	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
4054	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
4055	}
4056	# endif /* XAPIC_OFF_END */
4057
4058	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
4059
4060	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 102585

Download in other formats: